Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-6.5/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-t1hbh-rptxd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "281def36a75d8bd90a23f899d3cd7a663d74bf32" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-t1hbh" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git rev-list --no-walk 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 20 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=4199040f-7130-46fb-8af6-196e2bb540f3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-t1hbh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-de6e69a8ea75a42ea3ef5bc05e88104f5785c12a567deb6ce7101124d13f2fcf NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-t1hbh GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE="" GOARCH="amd64" GOBIN="" GOCACHE="/home/jenkins/.cache/go-build" GOENV="/home/jenkins/.config/go/env" GOEXE="" GOEXPERIMENT="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="linux" GOINSECURE="" GOMODCACHE="/go/pkg/mod" GONOPROXY="" GONOSUMDB="" GOOS="linux" GOPATH="/go" GOPRIVATE="" GOPROXY="http://goproxy.apps.svc,https://proxy.golang.org,direct" GOROOT="/usr/local/go" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" GOVCS="" GOVERSION="go1.19.13" GCCGO="gccgo" GOAMD64="v1" AR="ar" CC="gcc" CXX="g++" CGO_ENABLED="1" GOMOD="/dev/null" GOWORK="" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1070703144=/tmp/go-build -gno-record-gcc-switches" + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--w23h4 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 8890 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 9a8c490a53a7ae98 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Wed May 29 09:56:22 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11184/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11184/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11184/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11184/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/kv/client.go, cdc/kv/client_bench_test.go, cdc/kv/client_test.go, cdc/kv/metrics.go, cdc/kv/region_worker.go, cdc/processor/pipeline/puller.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager_test.go, cdc/processor/sinkmanager/table_sink_worker_test.go, cdc/processor/sourcemanager/manager.go, cdc/processor/sourcemanager/puller/puller_wrapper.go, cdc/puller/ddl_puller.go, cdc/puller/puller.go, cdc/puller/puller_test.go] [Pipeline] echo diff file not matched: cdc/kv/client.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-40df850-ef982f9) 208633856 bytes in 0.91 secs (229166744 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at dadfbcedf Merge commit 'ef982f9ef15da9a3f420632a794ec679c770ef45' into HEAD POST git-upload-pack (686 bytes) POST git-upload-pack (1023 bytes) POST git-upload-pack (gzip 1973 to 1048 bytes) From https://github.com/pingcap/tiflow 0daf56ac4..54e93ed06 release-6.5 -> origin/release-6.5 * [new ref] refs/pull/11184/head -> origin/pr/11184/head Warning: you are leaving 1 commit behind, not connected to any of your branches: dadfbcedf Merge commit 'ef982f9ef15da9a3f420632a794ec679c770ef45' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch dadfbcedf HEAD is now at 54e93ed06 syncpoint (ticdc): make syncpoint support base64 encoded password (#11162) ๐Ÿšง Checkouting to base SHA:54e93ed06ebc81ac717648679b252e8bd074ec39... HEAD is now at 54e93ed06 syncpoint (ticdc): make syncpoint support base64 encoded password (#11162) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 54e93ed06ebc81ac717648679b252e8bd074ec39 54e93ed06 syncpoint (ticdc): make syncpoint support base64 encoded password (#11162) 0ba932965 (redo)ticdc: fix the event orderliness in redo log (#11117) (#11180) f99c4dc3a *(ticdc): split old update kv entry after restarting changefeed (#10919) (#11028) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 54e93ed06ebc81ac717648679b252e8bd074ec39 ... Merge made by the 'ort' strategy. cdc/kv/client.go | 79 ++++------------------ cdc/kv/client_bench_test.go | 4 +- cdc/kv/client_test.go | 51 +++++++------- cdc/kv/metrics.go | 18 +---- cdc/kv/region_worker.go | 38 ++--------- cdc/processor/pipeline/puller.go | 1 - cdc/processor/processor.go | 2 +- cdc/processor/sinkmanager/manager_test.go | 2 +- .../sinkmanager/table_sink_worker_test.go | 2 +- cdc/processor/sourcemanager/manager.go | 22 +++--- .../sourcemanager/puller/puller_wrapper.go | 2 - cdc/puller/ddl_puller.go | 1 - cdc/puller/puller.go | 8 +-- cdc/puller/puller_test.go | 3 +- 14 files changed, 60 insertions(+), 173 deletions(-) ๐Ÿงพ Pre-merged result: 91afb776d8e607224e8f53cc50086ec346799afe 91afb776d Merge commit 'f3fe19e72a8f04977f85a144a37af82487376ff9' into HEAD 54e93ed06 syncpoint (ticdc): make syncpoint support base64 encoded password (#11162) f3fe19e72 fix error โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-54e93ed-f3fe19e) 206277632 bytes in 3.67 secs (56201360 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + chmod +x ../scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh + /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh Download binaries from branch release-6.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 379 0 --:--:-- --:--:-- --:--:-- 379 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 668 0 --:--:-- --:--:-- --:--:-- 672 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 795 0 --:--:-- --:--:-- --:--:-- 803 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1251 0 --:--:-- --:--:-- --:--:-- 1281 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9c5bf64d0dd123180a60642337956face1c1e3d/centos7/tidb-server.tar.gz 2024-05-29 17:57:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9c5bf64d0dd123180a60642337956face1c1e3d/centos7/tidb-server.tar.gz [501274259/501274259] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/33d839bdd7ed98fca9823febbddb43fcdc03a36a/centos7/pd-server.tar.gz 2024-05-29 17:57:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/33d839bdd7ed98fca9823febbddb43fcdc03a36a/centos7/pd-server.tar.gz [140293259/140293259] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/627d4a572b4d7d523370e5ed21ee67fee8d798a6/centos7/tikv-server.tar.gz 2024-05-29 17:58:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/627d4a572b4d7d523370e5ed21ee67fee8d798a6/centos7/tikv-server.tar.gz [723214974/723214974] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/61348e865b1b4b46013233aa3e210ea158a364ce/centos7/tiflash.tar.gz 2024-05-29 17:58:21 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/61348e865b1b4b46013233aa3e210ea158a364ce/centos7/tiflash.tar.gz [205615551/205615551] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-29 17:58:25 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-29 17:58:26 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-29 17:58:27 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-29 17:58:28 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-29 17:58:33 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] total 1.4G drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 17:58 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 29 17:58 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 29 17:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 29 17:58 jq lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M May 28 18:14 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M May 10 15:52 tikv-server Download SUCCESS + ls -alh ./bin/ total 1.4G drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 17:58 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 29 17:58 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 29 17:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 29 17:58 jq lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M May 28 18:14 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M May 10 15:52 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/minio + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.4G drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 17:58 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 29 17:58 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 29 17:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 29 17:58 jq lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M May 28 18:14 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M May 10 15:52 tikv-server + ./bin/tidb-server -V Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 + ./bin/tikv-server -V TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 17:58 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 29 17:58 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 09:58:35" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc ./cmd/cdc/main.go go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading google.golang.org/grpc v1.51.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/testify v1.8.4 go: downloading golang.org/x/time v0.2.0 go: downloading cloud.google.com/go v0.105.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/google/uuid v1.3.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/cenkalti/backoff/v4 v4.0.2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading google.golang.org/api v0.103.0 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/go-openapi/jsonreference v0.20.0 go: downloading github.com/go-openapi/spec v0.20.6 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/go-openapi/jsonpointer v0.19.5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading github.com/go-openapi/swag v0.21.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.24.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading github.com/google/gofuzz v1.1.0 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading github.com/go-logr/logr v1.2.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 10:01:04" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 10:01:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 10:01:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/chaos-mesh/go-sqlsmith v0.0.0-20220905074648-403033efad45 go: downloading github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0 go: downloading go.uber.org/dig v1.13.0 go: downloading gorm.io/gorm v1.23.8 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/glebarez/go-sqlite v1.17.3 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/glebarez/sqlite v1.4.6 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading go.uber.org/goleak v1.2.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.16.8 go: downloading modernc.org/sqlite v1.17.3 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.1.1 go: downloading modernc.org/mathutil v1.4.1 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 10:01:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g91afb776d" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-29 10:01:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=91afb776d8e607224e8f53cc50086ec346799afe" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 746M drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 18:04 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 29 17:58 .. -rwxr-xr-x. 1 jenkins jenkins 162M May 29 18:04 cdc -rwxr-xr-x. 1 jenkins jenkins 311M May 29 18:04 cdc.test -rwxr-xr-x. 1 jenkins jenkins 137M May 29 18:01 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 137M May 29 18:01 cdc_storage_consumer + ./bin/cdc version Release Version: v6.5.9-12-g91afb776d Git Commit Hash: 91afb776d8e607224e8f53cc50086ec346799afe Git Branch: HEAD UTC Build Time: 2024-05-29 10:01:10 Go Version: go version go1.19.13 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-54e93ed-f3fe19e) 781305344 bytes in 12.82 secs (60937722 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x. 2 jenkins jenkins 4.0K May 29 18:05 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 29 17:58 .. -rwxr-xr-x. 1 jenkins jenkins 162M May 29 18:04 cdc -rwxr-xr-x. 1 jenkins jenkins 311M May 29 18:04 cdc.test -rwxr-xr-x. 1 jenkins jenkins 137M May 29 18:01 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 137M May 29 18:01 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 17M May 29 18:05 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 29 18:05 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 29 18:05 jq lrwxrwxrwx. 1 jenkins jenkins 13 May 29 18:05 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 29 18:05 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 29 18:05 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 29 18:05 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 29 18:05 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 29 18:05 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 29 18:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M May 29 18:05 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 29 18:05 minio -rwxr-xr-x. 1 jenkins jenkins 36M May 29 18:05 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M May 29 18:05 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M May 29 18:05 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M May 29 18:05 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M May 29 18:05 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M May 29 18:05 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M May 29 18:05 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 29 18:05 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M May 29 18:05 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M May 29 18:05 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M May 29 18:05 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 42.94 secs (55934110 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G0') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G1') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G2') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G3') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G4') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G5') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G6') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G7') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G8') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G9') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G0') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G1') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G2') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G3') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G4') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G5') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G6') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G7') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G8') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G9') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh [Pipeline] { [Pipeline] node Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-c333r-bb7d5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bda81ef0713999b3d281ee93c816edb36e69ec58" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-c333r" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wpfjd-6hd8l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5722ee58d55d0a3aa75a1fd5620348bfd22b3936" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wpfjd" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-dxnld-c7q21 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "14f4ab5df5620141d6e5430619f897d522007361" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-dxnld" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" The recommended git tool is: git Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2f596fdb; decorates RemoteLauncher[hudson.remoting.Channel@6718e571:JNLP4-connect connection from 10.233.93.153/10.233.93.153:41000] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@69240de0; decorates RemoteLauncher[hudson.remoting.Channel@3bb26917:JNLP4-connect connection from 10.233.107.81/10.233.107.81:40490] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@652f7f35; decorates RemoteLauncher[hudson.remoting.Channel@1cc57b9d:JNLP4-connect connection from 10.233.69.175/10.233.69.175:35312] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-z2jh2-7j8c6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7a39fa2584c227552198d2fd1bfa59eec26a92c0" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-z2jh2" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 [Pipeline] node Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2f190d63; decorates RemoteLauncher[hudson.remoting.Channel@3352b892:JNLP4-connect connection from 10.233.70.39/10.233.70.39:52232] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" [Pipeline] withEnv [Pipeline] { Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-6rs0z-xccrf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f60f85e52e1ff563127501c13797852416080c2d" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-6rs0z" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] container [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) [Pipeline] checkout Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-frdvn-jlh24 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a1c86867ae6efdaed1695618012a0a11e366f093" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-frdvn" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] podTemplate [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 [Pipeline] node [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@639e9762; decorates RemoteLauncher[hudson.remoting.Channel@4463141e:JNLP4-connect connection from 10.233.100.23/10.233.100.23:49162] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] podTemplate Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] node [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage Avoid second fetch [Pipeline] { (Test) Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) [Pipeline] withCredentials Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1af711a4; decorates RemoteLauncher[hudson.remoting.Channel@601cd282:JNLP4-connect connection from 10.233.69.233/10.233.69.233:48370] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] container [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5217k-701j0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2c45fdddb5692ee294d5fdc5207d631ea663a160" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5217k" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-bh90h-gdzkm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "780459ae39c5e20c4b971add9764b4a4d2dda06c" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-bh90h" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-f5dx9-nlchm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f388aa0af5d673a687632fd4c7a4f7c104d48835" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-f5dx9" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5gmmq-rsmr5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c2a1569008a600dd97408cd0494c0d955c35a4b7" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5gmmq" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wqvgk-qfnh8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a76c580620081e1500b49d018335c7544b0da6d8" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wqvgk" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-vx72k-lwml0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e24d0ba2d9c3f52246cbc6fef58a08d34bf7c376" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-vx72k" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.47 secs (537646444 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] stage [Pipeline] { (Test) [Pipeline] { [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G0 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group0: api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=0372f815-0a6f-4a02-b443-fb9ae04650b0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G0 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-c333r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-c333r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-sjs4s-fsp5c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5fa36362e3ce6774d4decc893919031f866277f4" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-sjs4s" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-xrt1s-k0hlt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ba666b42566212f7e5485c399a5b0ed7803d9654" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-xrt1s" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-3md32-tgwq1 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7e70cf8ccb7dfeff1fc9f5179e78dab6638827b0" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-3md32" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-0p8km-57883 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "24a6fd35f793d0ee5a3d3cdef26393d794d99654" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-0p8km" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 6.70 secs (358687934 bytes/sec) [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:31.897 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:31.897 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432f840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:1024, start at 2024-05-29 18:06:31.909925141 +0800 CST m=+4.154721104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:31.916 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.02 secs (597568334 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G1 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group1: big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=d5abf774-e142-4582-b22b-cd72d43a09dc BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G1 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wpfjd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wpfjd pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] stage [Pipeline] { (Test) Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible [Pipeline] stage [Pipeline] { (Test) [Pipeline] { [Pipeline] { [Pipeline] { go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout go: downloading cloud.google.com/go v0.105.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/sys v0.14.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G2 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/google/uuid v1.3.0 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group2: capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=4562d797-90c1-4b74-a74b-77204dab4984 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G2 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-dxnld GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-dxnld GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@209580e; decorates RemoteLauncher[hudson.remoting.Channel@492c62b6:JNLP4-connect connection from 10.233.108.116/10.233.108.116:53042] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@de17b17; decorates RemoteLauncher[hudson.remoting.Channel@f492faa:JNLP4-connect connection from 10.233.107.31/10.233.107.31:59674] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@74df1c8b; decorates RemoteLauncher[hudson.remoting.Channel@7d26f5b5:JNLP4-connect connection from 10.233.88.207/10.233.88.207:36366] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@c02a183; decorates RemoteLauncher[hudson.remoting.Channel@7ce49b8a:JNLP4-connect connection from 10.233.93.152/10.233.93.152:54586] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7ef58bd; decorates RemoteLauncher[hudson.remoting.Channel@428e297f:JNLP4-connect connection from 10.233.71.61/10.233.71.61:36672] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@29563996; decorates RemoteLauncher[hudson.remoting.Channel@3fde94d1:JNLP4-connect connection from 10.233.106.78/10.233.106.78:60312] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Avoid second fetch Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Verifying downstream PD is started... Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.99 secs (480969168 bytes/sec) [Pipeline] { [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G4 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group4: clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=73c8d190-f264-48dd-9c3a-8c55bcebf493 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G4 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-z2jh2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-z2jh2 pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@51e2fb42; decorates RemoteLauncher[hudson.remoting.Channel@661db68:JNLP4-connect connection from 10.233.90.144/10.233.90.144:56706] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3233cc49; decorates RemoteLauncher[hudson.remoting.Channel@19e47e97:JNLP4-connect connection from 10.233.126.71/10.233.126.71:56224] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@56d0ee31; decorates RemoteLauncher[hudson.remoting.Channel@6883f968:JNLP4-connect connection from 10.233.69.183/10.233.69.183:43718] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@736057c3; decorates RemoteLauncher[hudson.remoting.Channel@398ff0a6:JNLP4-connect connection from 10.233.100.150/10.233.100.150:34672] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:47.638 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:47.638 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4425880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:1070, start at 2024-05-29 18:06:47.679391876 +0800 CST m=+4.669047233 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:47.687 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Verifying downstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" Starting CDC state checker... Avoid second fetch Checking out Revision 44e9fdd1b388d71f07c1fc94e6fccf31c512143e (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Commit message: "k8s docs: bump pdf stable version to release-1.6 (#2969)" go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/klauspost/compress v1.15.14 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/gogo/protobuf v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:51.481 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:51.481 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4461600017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:1022, start at 2024-05-29 18:06:51.51511165 +0800 CST m=+4.248802100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:51.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/klauspost/compress v1.15.14 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 8.58 secs (279798255 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:57.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:57.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44c1b8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:967, start at 2024-05-29 18:06:57.654793203 +0800 CST m=+4.263482816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:08:57.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading golang.org/x/time v0.2.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading cloud.google.com/go v0.105.0 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/sys v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 7.58 secs (317034361 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G3 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group3: changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=fa123e05-471e-48ac-893f-6ca781c285e1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G3 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-6rs0z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-6rs0z GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G5 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group5: consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=01394d7a-1008-4738-aab0-9691c6ab4dd4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G5 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-frdvn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-frdvn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) The 1 times to try to start tidb cluster... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] withCredentials start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.06 secs (592157321 bytes/sec) [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.648 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.648 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4649140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:960, start at 2024-05-29 18:07:22.732142008 +0800 CST m=+4.394136245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.326 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.326 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4642680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:1012, start at 2024-05-29 18:07:22.316016563 +0800 CST m=+5.804734929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:22.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/time v0.2.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/klauspost/compress v1.15.14 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading cloud.google.com/go v0.105.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 14.64 secs (164036005 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.41 secs (544255041 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 6.13 secs (391978047 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.70 secs (510808987 bytes/sec) [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G9 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group9: kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=ce2324e7-a3cc-4ccf-8c94-3c7baec8fa42 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G9 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5217k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5217k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:08:01 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G6 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group6: ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=5a113710-be4a-43ab-af4a-2bba14841451 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G6 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-bh90h GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-bh90h pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G10 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group10: many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=a01fdaf7-20fc-4f41-816c-05811af553da BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5gmmq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-5gmmq pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G7 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group7: default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=42a1dd2e-f8c6-44dd-b145-983402539133 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G7 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wqvgk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-wqvgk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:08:04 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G8 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group8: generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=b14fd699-70d8-4726-a8ed-0b1c8dae13a8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G8 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-f5dx9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-f5dx9 pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:08:08 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.91 secs (489042398 bytes/sec) [Pipeline] { [Pipeline] sh find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G11 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group11: multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=89c63716-68ba-480d-a5e8-8ec7434035b3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-vx72k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-vx72k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:15.396 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:15.396 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4981300002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:1063, start at 2024-05-29 18:08:15.436764926 +0800 CST m=+4.215032156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:15.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 5.36 secs (448341551 bytes/sec) [Pipeline] { [Pipeline] cache Starting CDC state checker... go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tinylib/msgp v1.1.6 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/docker/go-units v0.4.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading gopkg.in/yaml.v3 v3.0.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:18.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:18.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b94c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:1071, start at 2024-05-29 18:08:19.030683886 +0800 CST m=+4.506844889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:19.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:19.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:19.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:20.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:20.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:1042, start at 2024-05-29 18:08:20.33861383 +0800 CST m=+5.497258701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:20.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.349 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.349 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f6600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1299, start at 2024-05-29 18:08:22.971754984 +0800 CST m=+4.902150766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.984 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.787 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.787 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f4a4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1122, start at 2024-05-29 18:08:22.864370238 +0800 CST m=+4.105482443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/stretchr/testify v1.8.4 go: downloading golang.org/x/sync v0.5.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading cloud.google.com/go v0.105.0 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading golang.org/x/text v0.14.0 Starting CDC state checker... go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.46 secs (537935950 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading golang.org/x/time v0.2.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading cloud.google.com/go v0.105.0 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 4.81 secs (499713588 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:33.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:33.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a97b00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1126, start at 2024-05-29 18:08:33.307931062 +0800 CST m=+4.229835529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:33.314 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432f840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:1024, start at 2024-05-29 18:06:31.909925141 +0800 CST m=+4.154721104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc) 2401822720 bytes in 5.70 secs (421093284 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G13 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group13: processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=2f86e51a-85c7-44d6-9ea1-1fd58b4d93a1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-sjs4s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-sjs4s pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G12 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group12: owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=f68d3fa1-50f2-4fb6-9bcb-5a8349cf9855 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-xrt1s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-xrt1s GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G15 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group15: storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=60b5c9a4-2344-4611-8406-6728c70dac6c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-3md32 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-3md32 pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Wed May 29 18:08:43 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Wed May 29 18:08:43 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G14 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group14: sequence simple sink_hang sink_retry sorter split_region Sink type: mysql PROW_JOB_ID=77d1d4f8-28e5-40a8-84ac-29b8df33decd JENKINS_NODE_COOKIE=68946514-6422-437b-a228-6ea72a472d6a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1795755897746624512","prowjobid":"77d1d4f8-28e5-40a8-84ac-29b8df33decd","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-6.5","base_sha":"54e93ed06ebc81ac717648679b252e8bd074ec39","pulls":[{"number":11184,"author":"asddongmen","sha":"f3fe19e72a8f04977f85a144a37af82487376ff9","title":"kvClient (ticdc): revert e5999e3 to remove useless metrics"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#252 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1795755897746624512 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/252/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-0p8km GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_252-0p8km pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=252 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4425880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:1070, start at 2024-05-29 18:06:47.679391876 +0800 CST m=+4.669047233 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:55.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:55.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf5e8001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:1014, start at 2024-05-29 18:08:55.722606534 +0800 CST m=+4.373486816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:55.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:56.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:56.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c07f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:1020, start at 2024-05-29 18:08:56.867566852 +0800 CST m=+4.569252393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:56.876 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44c1b8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:967, start at 2024-05-29 18:06:57.654793203 +0800 CST m=+4.263482816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432ed80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:949, start at 2024-05-29 18:06:31.889722995 +0800 CST m=+4.192063568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f432f840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:1024, start at 2024-05-29 18:06:31.909925141 +0800 CST m=+4.154721104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:31.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:31.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:31.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading cloud.google.com/go v0.105.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2d400001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1111, start at 2024-05-29 18:08:59.216279361 +0800 CST m=+4.332182952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:59.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.uber.org/multierr v1.8.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 Starting CDC state checker... go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading golang.org/x/text v0.14.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:02.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:02.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5c680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:1064, start at 2024-05-29 18:09:02.235025081 +0800 CST m=+4.391713022 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:02.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dustin/go-humanize v1.0.0 start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.uber.org/atomic v1.10.0 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading golang.org/x/time v0.2.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4461600017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:1022, start at 2024-05-29 18:06:51.51511165 +0800 CST m=+4.248802100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/golang/glog v1.0.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4424b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:980, start at 2024-05-29 18:06:47.628672485 +0800 CST m=+4.678816471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4425880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:1070, start at 2024-05-29 18:06:47.679391876 +0800 CST m=+4.669047233 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:47.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:47.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:47.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:09:15 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2358823590.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:09:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.23656.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44bf800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:878, start at 2024-05-29 18:06:57.532720554 +0800 CST m=+4.215772136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f44c1b8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:967, start at 2024-05-29 18:06:57.654793203 +0800 CST m=+4.263482816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:57.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:57.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:57.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:09:19 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.2349023492.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='450095312729276419 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095312729276419 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:09:19 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2369123693.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:09:19 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/819c65f5-1b6d-4a16-a85a-c0988de71e3b {"id":"819c65f5-1b6d-4a16-a85a-c0988de71e3b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d320018e 819c65f5-1b6d-4a16-a85a-c0988de71e3b /tidb/cdc/default/default/upstream/7374360917998318964 {"id":7374360917998318964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.23714.out cli changefeed create --start-ts=450095312729276419 '--sink-uri=mysql://normal:123456@127.0.0.1:3306?transaction-atomicity=none' Create changefeed successfully! ID: 829b7e49-197d-4ec4-aecd-bb34cfb2620c Info: {"upstream_id":7374360917998318964,"namespace":"default","id":"829b7e49-197d-4ec4-aecd-bb34cfb2620c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306?transaction-atomicity=none","create_time":"2024-05-29T18:09:19.51471772+08:00","start_ts":450095312729276419,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095312729276419,"checkpoint_ts":450095312729276419,"checkpoint_time":"2024-05-29 18:09:18.739"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:09:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7055caf3-eb5d-4bf9-ae9c-1655993c4a20 {"id":"7055caf3-eb5d-4bf9-ae9c-1655993c4a20","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d32ef28e 7055caf3-eb5d-4bf9-ae9c-1655993c4a20 /tidb/cdc/default/default/upstream/7374360964503082041 {"id":7374360964503082041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7055caf3-eb5d-4bf9-ae9c-1655993c4a20 {"id":"7055caf3-eb5d-4bf9-ae9c-1655993c4a20","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d32ef28e 7055caf3-eb5d-4bf9-ae9c-1655993c4a20 /tidb/cdc/default/default/upstream/7374360964503082041 {"id":7374360964503082041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7055caf3-eb5d-4bf9-ae9c-1655993c4a20 {"id":"7055caf3-eb5d-4bf9-ae9c-1655993c4a20","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d32ef28e 7055caf3-eb5d-4bf9-ae9c-1655993c4a20 /tidb/cdc/default/default/upstream/7374360964503082041 {"id":7374360964503082041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: fa0e5e9c-286f-4afa-a958-342e9884ecca Info: {"upstream_id":7374360964503082041,"namespace":"default","id":"fa0e5e9c-286f-4afa-a958-342e9884ecca","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:09:22.348802872+08:00","start_ts":450095312823648257,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095312823648257,"checkpoint_ts":450095312823648257,"checkpoint_time":"2024-05-29 18:09:19.099"} # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4460b0001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:931, start at 2024-05-29 18:06:51.473685302 +0800 CST m=+4.264156999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4461600017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:1022, start at 2024-05-29 18:06:51.51511165 +0800 CST m=+4.248802100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:09:51.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:06:51.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:56:51.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4642680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:1012, start at 2024-05-29 18:07:22.316016563 +0800 CST m=+5.804734929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.369 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4649140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:960, start at 2024-05-29 18:07:22.732142008 +0800 CST m=+4.394136245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f464304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:920, start at 2024-05-29 18:07:22.317414354 +0800 CST m=+5.906255137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4642680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:1012, start at 2024-05-29 18:07:22.316016563 +0800 CST m=+5.804734929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.369 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:09:51 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2360623608.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:09:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 {"id":"53b61bd2-5935-4f6a-8cd7-9767e4a0edd7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b5198e 53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 {"id":"53b61bd2-5935-4f6a-8cd7-9767e4a0edd7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b5198e 53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 {"id":"53b61bd2-5935-4f6a-8cd7-9767e4a0edd7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b5198e 53b61bd2-5935-4f6a-8cd7-9767e4a0edd7 /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4647f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:870, start at 2024-05-29 18:07:22.641278972 +0800 CST m=+4.355046476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4649140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:960, start at 2024-05-29 18:07:22.732142008 +0800 CST m=+4.394136245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:10:22.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:07:22.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:57:22.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.24361.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095322856947713 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095322856947713 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:09:59 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.2251822520.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5af2650-d93f-40cb-95bb-98bf01635394 {"id":"c5af2650-d93f-40cb-95bb-98bf01635394","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d2e6480a c5af2650-d93f-40cb-95bb-98bf01635394 /tidb/cdc/default/default/upstream/7374360855627413298 {"id":7374360855627413298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5af2650-d93f-40cb-95bb-98bf01635394 {"id":"c5af2650-d93f-40cb-95bb-98bf01635394","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d2e6480a c5af2650-d93f-40cb-95bb-98bf01635394 /tidb/cdc/default/default/upstream/7374360855627413298 {"id":7374360855627413298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5af2650-d93f-40cb-95bb-98bf01635394 {"id":"c5af2650-d93f-40cb-95bb-98bf01635394","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d2e6480a c5af2650-d93f-40cb-95bb-98bf01635394 /tidb/cdc/default/default/upstream/7374360855627413298 {"id":7374360855627413298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7374360855627413298,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-05-29T18:10:02.314066617+08:00","start_ts":450095324134637571,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095324134637571,"checkpoint_ts":450095324134637571,"checkpoint_time":"2024-05-29 18:10:02.247"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7374360855627413298, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450095324134637571, "checkpoint_time": "2024-05-29 18:10:02.247", "error": null }' + echo '{ "upstream_id": 7374360855627413298, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450095324134637571, "checkpoint_time": "2024-05-29 18:10:02.247", "error": null }' { "upstream_id": 7374360855627413298, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450095324134637571, "checkpoint_time": "2024-05-29 18:10:02.247", "error": null } ++ echo '{' '"upstream_id":' 7374360855627413298, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 450095324134637571, '"checkpoint_time":' '"2024-05-29' '18:10:02.247",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374360855627413298, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 450095324134637571, '"checkpoint_time":' '"2024-05-29' '18:10:02.247",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/cenkalti/backoff/v4 v4.0.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading k8s.io/api v0.24.0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/locales v0.14.1 go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading github.com/go-logr/logr v1.2.0 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ***************** properties ***************** "readallfields"="true" "readproportion"="0" "updateproportion"="0" "threadcount"="1" "scanproportion"="0" "mysql.db"="big_txn" "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "requestdistribution"="uniform" "insertproportion"="0" "mysql.port"="4000" "dotransactions"="false" "recordcount"="5000" "mysql.user"="root" "fieldcount"="100" ********************************************** # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 10.0, Count: 2370, OPS: 237.2, Avg(us): 3913, Min(us): 1583, Max(us): 89136, 95th(us): 4000, 99th(us): 78000 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4981300002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:1063, start at 2024-05-29 18:08:15.436764926 +0800 CST m=+4.215032156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.464 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f4a4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1122, start at 2024-05-29 18:08:22.864370238 +0800 CST m=+4.105482443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.885 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f6600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1299, start at 2024-05-29 18:08:22.971754984 +0800 CST m=+4.902150766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:23.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table clustered_index_test.t0 exists table clustered_index_test.t1 not exists for 1-th check, retry later table clustered_index_test.t1 exists table clustered_index_test.t2 not exists for 1-th check, retry later table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b94c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:1071, start at 2024-05-29 18:08:19.030683886 +0800 CST m=+4.506844889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:19.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:19.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:1042, start at 2024-05-29 18:08:20.33861383 +0800 CST m=+5.497258701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.431 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.349 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.349 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 7-th time, retry later INSERT - Takes(s): 20.0, Count: 4654, OPS: 232.8, Avg(us): 4030, Min(us): 1583, Max(us): 92584, 95th(us): 5000, 99th(us): 74000 Run finished, takes 21.182416091s INSERT - Takes(s): 21.2, Count: 5000, OPS: 236.1, Avg(us): 3971, Min(us): 1583, Max(us): 92584, 95th(us): 4000, 99th(us): 74000 table big_txn.usertable exists start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 8-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.24462.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095331129163777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095331129163777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:10:29 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2450224504.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40c8a53-db6c-47c5-a588-3195a11cf516 {"id":"d40c8a53-db6c-47c5-a588-3195a11cf516","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d35cf41c d40c8a53-db6c-47c5-a588-3195a11cf516 /tidb/cdc/default/default/upstream/7374360934200515169 {"id":7374360934200515169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40c8a53-db6c-47c5-a588-3195a11cf516 {"id":"d40c8a53-db6c-47c5-a588-3195a11cf516","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d35cf41c d40c8a53-db6c-47c5-a588-3195a11cf516 /tidb/cdc/default/default/upstream/7374360934200515169 {"id":7374360934200515169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40c8a53-db6c-47c5-a588-3195a11cf516 {"id":"d40c8a53-db6c-47c5-a588-3195a11cf516","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d35cf41c d40c8a53-db6c-47c5-a588-3195a11cf516 /tidb/cdc/default/default/upstream/7374360934200515169 {"id":7374360934200515169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x lease 22318fc3d35cf41c revoked table capture_session_done_during_task.t exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 9-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:10:41 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> check diff failed 10-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a97b00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1126, start at 2024-05-29 18:08:33.307931062 +0800 CST m=+4.229835529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 11-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 12-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1214, start at 2024-05-29 18:08:22.808055977 +0800 CST m=+4.825346891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f6600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:1299, start at 2024-05-29 18:08:22.971754984 +0800 CST m=+4.902150766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:23.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:10:47 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2370523707.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f497fbc001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:971, start at 2024-05-29 18:08:15.390112059 +0800 CST m=+4.263576564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4981300002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:1063, start at 2024-05-29 18:08:15.436764926 +0800 CST m=+4.215032156 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:15.464 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:15.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:15.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.23921.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095336150007809 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095336150007809 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:10:48 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2396023962.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 13-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 14-th time, retry later Starting Upstream TiDB... ***************** properties ***************** "requestdistribution"="uniform" "fieldcount"="100" "mysql.port"="4000" "workload"="core" "insertproportion"="0" "operationcount"="0" "mysql.user"="root" "mysql.db"="consistent_replicate_storage_file" "scanproportion"="0" "readallfields"="true" "recordcount"="5000" "mysql.host"="127.0.0.1" "updateproportion"="0" "readproportion"="0" "threadcount"="10" "dotransactions"="false" ********************************************** Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d50d689d-049a-4c49-b182-74c35c8326d7 {"id":"d50d689d-049a-4c49-b182-74c35c8326d7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c48e d50d689d-049a-4c49-b182-74c35c8326d7 /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d50d689d-049a-4c49-b182-74c35c8326d7 {"id":"d50d689d-049a-4c49-b182-74c35c8326d7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c48e d50d689d-049a-4c49-b182-74c35c8326d7 /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d50d689d-049a-4c49-b182-74c35c8326d7 {"id":"d50d689d-049a-4c49-b182-74c35c8326d7","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c48e d50d689d-049a-4c49-b182-74c35c8326d7 /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info: {"upstream_id":7374361327621511023,"namespace":"default","id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:10:50.971667859+08:00","start_ts":450095336870379526,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095336870379526,"checkpoint_ts":450095336870379526,"checkpoint_time":"2024-05-29 18:10:50.830"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f3e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1027, start at 2024-05-29 18:08:22.781165362 +0800 CST m=+4.074450444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49f4a4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:1122, start at 2024-05-29 18:08:22.864370238 +0800 CST m=+4.105482443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:22.885 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:22.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:22.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.22170.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='450095336994111491 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095336994111491 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:10:51 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.2220022202.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0be56f6-f231-4738-b88d-9840b51c2b64 {"id":"c0be56f6-f231-4738-b88d-9840b51c2b64","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4898d8f c0be56f6-f231-4738-b88d-9840b51c2b64 /tidb/cdc/default/default/upstream/7374361301005668143 {"id":7374361301005668143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0be56f6-f231-4738-b88d-9840b51c2b64 {"id":"c0be56f6-f231-4738-b88d-9840b51c2b64","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4898d8f c0be56f6-f231-4738-b88d-9840b51c2b64 /tidb/cdc/default/default/upstream/7374361301005668143 {"id":7374361301005668143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0be56f6-f231-4738-b88d-9840b51c2b64 {"id":"c0be56f6-f231-4738-b88d-9840b51c2b64","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4898d8f c0be56f6-f231-4738-b88d-9840b51c2b64 /tidb/cdc/default/default/upstream/7374361301005668143 {"id":7374361301005668143,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24021.out cli changefeed create --start-ts=450095336150007809 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: ee6213d9-caae-4866-be1a-3b29ae006d47 Info: {"upstream_id":7374361301005668143,"namespace":"default","id":"ee6213d9-caae-4866-be1a-3b29ae006d47","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-29T18:10:51.797764168+08:00","start_ts":450095336150007809,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095336150007809,"checkpoint_ts":450095336150007809,"checkpoint_time":"2024-05-29 18:10:48.082"} PASS start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 15-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:958, start at 2024-05-29 18:08:20.332949323 +0800 CST m=+5.591980925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49cd2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:1042, start at 2024-05-29 18:08:20.33861383 +0800 CST m=+5.497258701 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:20.431 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:20.349 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:20.349 +0800 All versions after safe point can be accessed. (DO NOT EDIT) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.23384.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450095337425338369 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095337425338369 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:10:53 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.2343223434.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 16-th time, retry later pass test: get status pass test: health [2024/05/29 18:10:48.660 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] pass test: changefeed apis [2024/05/29 18:10:52.310 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] Run finished, takes 4.582356731s INSERT - Takes(s): 4.6, Count: 5000, OPS: 1094.9, Avg(us): 8352, Min(us): 1654, Max(us): 96993, 95th(us): 61000, 99th(us): 90000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688b6c6-9d28-4853-9238-21ef29e0aead {"id":"5688b6c6-9d28-4853-9238-21ef29e0aead","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4984a8e 5688b6c6-9d28-4853-9238-21ef29e0aead /tidb/cdc/default/default/upstream/7374361334093540873 {"id":7374361334093540873,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688b6c6-9d28-4853-9238-21ef29e0aead {"id":"5688b6c6-9d28-4853-9238-21ef29e0aead","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4984a8e 5688b6c6-9d28-4853-9238-21ef29e0aead /tidb/cdc/default/default/upstream/7374361334093540873 {"id":7374361334093540873,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688b6c6-9d28-4853-9238-21ef29e0aead {"id":"5688b6c6-9d28-4853-9238-21ef29e0aead","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4984a8e 5688b6c6-9d28-4853-9238-21ef29e0aead /tidb/cdc/default/default/upstream/7374361334093540873 {"id":7374361334093540873,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.22252.out cli changefeed create --start-ts=450095336994111491 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 27679f92-154c-4d27-8846-e87d3dcab3ca Info: {"upstream_id":7374361334093540873,"namespace":"default","id":"27679f92-154c-4d27-8846-e87d3dcab3ca","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:10:54.955067722+08:00","start_ts":450095336994111491,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095336994111491,"checkpoint_ts":450095336994111491,"checkpoint_time":"2024-05-29 18:10:51.302"} PASS pass test: delete changefeed apis VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b7b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:982, start at 2024-05-29 18:08:18.94446353 +0800 CST m=+4.521846681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:18.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:18.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f49b94c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:1071, start at 2024-05-29 18:08:19.030683886 +0800 CST m=+4.506844889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:19.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:19.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:19.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.23833.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + cleanup_process cdc.test + set +x + tso='450095338052648964 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095338052648964 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:10:55 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.2387823880.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit ++ date + echo '[Wed May 29 18:10:55 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Wed May 29 18:10:55 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de94405e-4901-4af7-a759-20b64f4df381 {"id":"de94405e-4901-4af7-a759-20b64f4df381","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4a3168f de94405e-4901-4af7-a759-20b64f4df381 /tidb/cdc/default/default/upstream/7374361316531331970 {"id":7374361316531331970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de94405e-4901-4af7-a759-20b64f4df381 {"id":"de94405e-4901-4af7-a759-20b64f4df381","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4a3168f de94405e-4901-4af7-a759-20b64f4df381 /tidb/cdc/default/default/upstream/7374361316531331970 {"id":7374361316531331970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de94405e-4901-4af7-a759-20b64f4df381 {"id":"de94405e-4901-4af7-a759-20b64f4df381","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4a3168f de94405e-4901-4af7-a759-20b64f4df381 /tidb/cdc/default/default/upstream/7374361316531331970 {"id":7374361316531331970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.23496.out cli changefeed create --start-ts=450095337425338369 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 9d1630b2-8293-4e39-9135-5341a457af6a Info: {"upstream_id":7374361316531331970,"namespace":"default","id":"9d1630b2-8293-4e39-9135-5341a457af6a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:10:56.660823108+08:00","start_ts":450095337425338369,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095337425338369,"checkpoint_ts":450095337425338369,"checkpoint_time":"2024-05-29 18:10:52.947"} PASS check diff failed 17-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 18-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7da159d-8355-4388-8de4-e7f67690c887 {"id":"e7da159d-8355-4388-8de4-e7f67690c887","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4aa018e e7da159d-8355-4388-8de4-e7f67690c887 /tidb/cdc/default/default/upstream/7374361316634465053 {"id":7374361316634465053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7da159d-8355-4388-8de4-e7f67690c887 {"id":"e7da159d-8355-4388-8de4-e7f67690c887","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4aa018e e7da159d-8355-4388-8de4-e7f67690c887 /tidb/cdc/default/default/upstream/7374361316634465053 {"id":7374361316634465053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7da159d-8355-4388-8de4-e7f67690c887 {"id":"e7da159d-8355-4388-8de4-e7f67690c887","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4aa018e e7da159d-8355-4388-8de4-e7f67690c887 /tidb/cdc/default/default/upstream/7374361316634465053 {"id":7374361316634465053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.23937.out cli changefeed create --start-ts=450095338052648964 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a6ee7e3b-81fa-48aa-831e-4aa73be3a11c Info: {"upstream_id":7374361316634465053,"namespace":"default","id":"a6ee7e3b-81fa-48aa-831e-4aa73be3a11c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:10:59.13971178+08:00","start_ts":450095338052648964,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095338052648964,"checkpoint_ts":450095338052648964,"checkpoint_time":"2024-05-29 18:10:55.340"} PASS # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 19-th time, retry later table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2d400001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1111, start at 2024-05-29 18:08:59.216279361 +0800 CST m=+4.332182952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 20-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf5e8001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:1014, start at 2024-05-29 18:08:55.722606534 +0800 CST m=+4.373486816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 21-th time, retry later [Wed May 29 18:10:54 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.2446924471.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:10:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 {"id":"6eda8d9e-4b8f-4be6-9a9f-b4968a515e54","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3c56e03 6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 /tidb/cdc/default/default/upstream/7374361065465035465 {"id":7374361065465035465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 {"id":"6eda8d9e-4b8f-4be6-9a9f-b4968a515e54","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3c56e03 6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 /tidb/cdc/default/default/upstream/7374361065465035465 {"id":7374361065465035465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 {"id":"6eda8d9e-4b8f-4be6-9a9f-b4968a515e54","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3c56e03 6eda8d9e-4b8f-4be6-9a9f-b4968a515e54 /tidb/cdc/default/default/upstream/7374361065465035465 {"id":7374361065465035465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.24526.out cli changefeed create --start-ts=450095322856947713 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7374361065465035465,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:10:57.8624476+08:00","start_ts":450095322856947713,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095322856947713,"checkpoint_ts":450095322856947713,"checkpoint_time":"2024-05-29 18:09:57.373"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7374361065465035465, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:10:57.867383776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7374361065465035465, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:10:57.867383776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7374361065465035465, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:10:57.867383776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374361065465035465, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-29T18:10:57.867383776+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374361065465035465, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-29T18:10:57.867383776+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:10:59 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cc9c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25781, start at 2024-05-29 18:11:04.115894989 +0800 CST m=+4.491746859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.128 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c07f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:1020, start at 2024-05-29 18:08:56.867566852 +0800 CST m=+4.569252393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 22-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a9aec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1044, start at 2024-05-29 18:08:33.475784975 +0800 CST m=+4.448612478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4a97b00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:1126, start at 2024-05-29 18:08:33.307931062 +0800 CST m=+4.229835529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:33.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:33.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:33.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.23298.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095341128646657 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095341128646657 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:11:07 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.2334123343.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later Starting CDC state checker... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later check diff failed 23-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/267a7565-837b-4381-9af1-9e3d80d0a21b {"id":"267a7565-837b-4381-9af1-9e3d80d0a21b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4d26090 267a7565-837b-4381-9af1-9e3d80d0a21b /tidb/cdc/default/default/upstream/7374361370717276970 {"id":7374361370717276970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/267a7565-837b-4381-9af1-9e3d80d0a21b {"id":"267a7565-837b-4381-9af1-9e3d80d0a21b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4d26090 267a7565-837b-4381-9af1-9e3d80d0a21b /tidb/cdc/default/default/upstream/7374361370717276970 {"id":7374361370717276970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/267a7565-837b-4381-9af1-9e3d80d0a21b {"id":"267a7565-837b-4381-9af1-9e3d80d0a21b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d4d26090 267a7565-837b-4381-9af1-9e3d80d0a21b /tidb/cdc/default/default/upstream/7374361370717276970 {"id":7374361370717276970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.23399.out cli changefeed create --start-ts=450095341128646657 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Create changefeed successfully! ID: c2233de4-cde4-48eb-bdfb-6b5644fa78ce Info: {"upstream_id":7374361370717276970,"namespace":"default","id":"c2233de4-cde4-48eb-bdfb-6b5644fa78ce","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:11:10.830578291+08:00","start_ts":450095341128646657,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095341128646657,"checkpoint_ts":450095341128646657,"checkpoint_time":"2024-05-29 18:11:07.074"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later + set +x check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:11:11 CST 2024] <<<<<< run test case clustered_index success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5c680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:1064, start at 2024-05-29 18:09:02.235025081 +0800 CST m=+4.391713022 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2b740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1022, start at 2024-05-29 18:08:59.129646924 +0800 CST m=+4.335671568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c2d400001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:1111, start at 2024-05-29 18:08:59.216279361 +0800 CST m=+4.332182952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:59.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:59.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:59.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:11:22 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.2018920191.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf520000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:926, start at 2024-05-29 18:08:55.642511672 +0800 CST m=+4.384972765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.624 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.624 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4bf5e8001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:1014, start at 2024-05-29 18:08:55.722606534 +0800 CST m=+4.373486816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:55.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:55.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:55.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:11:22 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.2488024882.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f23c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28564, start at 2024-05-29 18:11:22.931715412 +0800 CST m=+4.467770634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:22.939 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:22.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:22.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f554001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25708, start at 2024-05-29 18:11:23.140442537 +0800 CST m=+4.562226122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c08080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:917, start at 2024-05-29 18:08:56.853563553 +0800 CST m=+4.632618732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c07f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:1020, start at 2024-05-29 18:08:56.867566852 +0800 CST m=+4.569252393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:11:56.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:08:56.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:58:56.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:11:25 CST 2024] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.2478224784.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:25 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71212276-379c-4b6a-a8d7-c0eed2231ded {"id":"71212276-379c-4b6a-a8d7-c0eed2231ded","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5194d8e 71212276-379c-4b6a-a8d7-c0eed2231ded /tidb/cdc/default/default/upstream/7374361473765130947 {"id":7374361473765130947,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71212276-379c-4b6a-a8d7-c0eed2231ded {"id":"71212276-379c-4b6a-a8d7-c0eed2231ded","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5194d8e 71212276-379c-4b6a-a8d7-c0eed2231ded /tidb/cdc/default/default/upstream/7374361473765130947 {"id":7374361473765130947,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71212276-379c-4b6a-a8d7-c0eed2231ded {"id":"71212276-379c-4b6a-a8d7-c0eed2231ded","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5194d8e 71212276-379c-4b6a-a8d7-c0eed2231ded /tidb/cdc/default/default/upstream/7374361473765130947 {"id":7374361473765130947,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:25 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e6dea29-d5fd-4628-8ec7-32ad071dbf6b {"id":"1e6dea29-d5fd-4628-8ec7-32ad071dbf6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d50ed390 1e6dea29-d5fd-4628-8ec7-32ad071dbf6b /tidb/cdc/default/default/upstream/7374361487717959654 {"id":7374361487717959654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e6dea29-d5fd-4628-8ec7-32ad071dbf6b {"id":"1e6dea29-d5fd-4628-8ec7-32ad071dbf6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d50ed390 1e6dea29-d5fd-4628-8ec7-32ad071dbf6b /tidb/cdc/default/default/upstream/7374361487717959654 {"id":7374361487717959654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e6dea29-d5fd-4628-8ec7-32ad071dbf6b {"id":"1e6dea29-d5fd-4628-8ec7-32ad071dbf6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d50ed390 1e6dea29-d5fd-4628-8ec7-32ad071dbf6b /tidb/cdc/default/default/upstream/7374361487717959654 {"id":7374361487717959654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.20243.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: 9e7c8718-2996-4c4f-a41e-7dc15bbbd59c Info: {"upstream_id":7374361487717959654,"namespace":"default","id":"9e7c8718-2996-4c4f-a41e-7dc15bbbd59c","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-29T18:11:25.433256378+08:00","start_ts":450095345920901122,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095345920901122,"checkpoint_ts":450095345920901122,"checkpoint_time":"2024-05-29 18:11:25.355"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01f8cbf4-ce38-43be-863c-b5c4b0a57d8b {"id":"01f8cbf4-ce38-43be-863c-b5c4b0a57d8b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d521fb8e 01f8cbf4-ce38-43be-863c-b5c4b0a57d8b /tidb/cdc/default/default/upstream/7374361473351776179 {"id":7374361473351776179,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01f8cbf4-ce38-43be-863c-b5c4b0a57d8b {"id":"01f8cbf4-ce38-43be-863c-b5c4b0a57d8b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d521fb8e 01f8cbf4-ce38-43be-863c-b5c4b0a57d8b /tidb/cdc/default/default/upstream/7374361473351776179 {"id":7374361473351776179,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01f8cbf4-ce38-43be-863c-b5c4b0a57d8b {"id":"01f8cbf4-ce38-43be-863c-b5c4b0a57d8b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d521fb8e 01f8cbf4-ce38-43be-863c-b5c4b0a57d8b /tidb/cdc/default/default/upstream/7374361473351776179 {"id":7374361473351776179,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.24847.out cli changefeed create --start-ts=450095345829150721 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: c128a517-6f22-483d-9905-9020bf17b798 Info: {"upstream_id":7374361473351776179,"namespace":"default","id":"c128a517-6f22-483d-9905-9020bf17b798","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:11:28.428027915+08:00","start_ts":450095345829150721,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095345829150721,"checkpoint_ts":450095345829150721,"checkpoint_time":"2024-05-29 18:11:25.005"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cc9c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25781, start at 2024-05-29 18:11:04.115894989 +0800 CST m=+4.491746859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.128 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting CDC state checker... table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583dc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25345, start at 2024-05-29 18:11:32.24366434 +0800 CST m=+4.259939321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log table consistent_replicate_storage_file.t2 exists check diff successfully Starting CDC state checker... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:11:36 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2403824040.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/fd512f00-2883-40fa-9f5e-881b3b91c56c {ClusterID:default ID:default/fd512f00-2883-40fa-9f5e-881b3b91c56c Info:{"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} Status:0xc0033eb458 TaskPositions:map[8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 91, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 87, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 92, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 82, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 89, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 88, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 84, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d {"id":"8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b51b84 8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d /tidb/cdc/default/default/changefeed/info/fd512f00-2883-40fa-9f5e-881b3b91c56c {"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} /tidb/cdc/default/default/changefeed/status/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":450095349307801603,"min-table-barrier-ts":450095349307801604,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/fd512f00-2883-40fa-9f5e-881b3b91c56c {ClusterID:default ID:default/fd512f00-2883-40fa-9f5e-881b3b91c56c Info:{"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} Status:0xc0033eb458 TaskPositions:map[8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 91, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 87, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 92, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 82, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 89, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 88, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 84, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d {"id":"8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b51b84 8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d /tidb/cdc/default/default/changefeed/info/fd512f00-2883-40fa-9f5e-881b3b91c56c {"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} /tidb/cdc/default/default/changefeed/status/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":450095349307801603,"min-table-barrier-ts":450095349307801604,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/fd512f00-2883-40fa-9f5e-881b3b91c56c {ClusterID:default ID:default/fd512f00-2883-40fa-9f5e-881b3b91c56c Info:{"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} Status:0xc0033eb458 TaskPositions:map[8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 91, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 87, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 92, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 82, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 89, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 88, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating tableID: 84, resolvedTs: 450095349307801603, checkpointTs: 450095349307801603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d {"id":"8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d3b51b84 8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d /tidb/cdc/default/default/changefeed/info/fd512f00-2883-40fa-9f5e-881b3b91c56c {"upstream-id":7374361063914172992,"namespace":"default","changefeed-id":"fd512f00-2883-40fa-9f5e-881b3b91c56c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:09:54.209536181+08:00","start-ts":450095321992921097,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095322018611206} /tidb/cdc/default/default/changefeed/status/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":450095349307801603,"min-table-barrier-ts":450095349307801604,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ebf50f3-5b6f-4e59-9e79-a3aedd251c3d/fd512f00-2883-40fa-9f5e-881b3b91c56c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361063914172992 {"id":7374361063914172992,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5ad40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:978, start at 2024-05-29 18:09:02.163164475 +0800 CST m=+4.385887286 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f4c5c680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:1064, start at 2024-05-29 18:09:02.235025081 +0800 CST m=+4.391713022 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:12:02.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:09:02.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-17:59:02.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:11:40 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sinkv2/eventsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2211322115.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.22175.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 table big_txn.usertable1 exists Create changefeed successfully! ID: b8581dba-e847-4a1f-b10d-269632b5db2a Info: {"upstream_id":7374361490244667338,"namespace":"default","id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:11:43.633374076+08:00","start_ts":450095350685106181,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095350685106181,"checkpoint_ts":450095350685106181,"checkpoint_time":"2024-05-29 18:11:43.529"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table kill_owner_with_ddl.t1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table kill_owner_with_ddl.t1 exists /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (23710) - No such process wait process 23710 exit for 1-th time... process 23710 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:11:49 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2398823990.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cb280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25695, start at 2024-05-29 18:11:04.040703183 +0800 CST m=+4.501694598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f53cc9c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:25781, start at 2024-05-29 18:11:04.115894989 +0800 CST m=+4.491746859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:04.128 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:04.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:04.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:11:50 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2975529757.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:11:49.149 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/29 18:11:49.149 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/29 18:11:49.244 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd8975c9e_f704_4482_ac13_60d115e6a0fc"] [2024/05/29 18:11:49.399 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs53c5119e_f804_483b_9961_fb695b604dec"] [2024/05/29 18:11:49.525 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/29 18:11:49.534 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsed750c54_653f_49a2_bf30_a2cfdd6c7b95"] [2024/05/29 18:11:49.633 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs60286560_b800_4229_9f91_6028a04be5c8"] [2024/05/29 18:11:49.656 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/29 18:11:49.666 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:49.724 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs90bded9f_30c1_494a_916a_d4ab4c114010"] [2024/05/29 18:11:49.725 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsea61857d_257e_4dd5_8768_8639ac0bfcd5"] [2024/05/29 18:11:49.726 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd026c899_3700_4ad3_9163_67526b79deb5"] [2024/05/29 18:11:49.727 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs890674ed_4184_4a06_9efb_5064b28bd5fc"] [2024/05/29 18:11:49.731 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.127 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.128 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.144 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/29 18:11:50.346 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/29 18:11:50.350 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.430 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.542 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.549 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/29 18:11:50.638 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.726 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.726 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:50.731 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.731 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.734 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:50.738 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.739 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:11:50.743 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.745 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:50.749 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:11:51.040 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/29 18:11:51.045 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/29 18:11:51.056 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.133 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.136 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/29 18:11:51.335 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/29 18:11:51.348 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.351 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.524 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.626 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.646 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.648 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:51.724 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.725 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:51.726 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:11:51.742 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.746 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:51.749 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.751 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.756 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:11:51.854 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/29 18:11:51.855 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/29 18:11:52.031 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.045 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/29 18:11:52.140 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.328 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/29 18:11:52.334 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.431 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.440 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/29 18:11:52.441 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0012b9e90 TaskPositions:map[bef21930-677c-4beb-bb20-53c4f7fcd60a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095352887377925, checkpointTs: 450095352887377922, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bef21930-677c-4beb-bb20-53c4f7fcd60a {"id":"bef21930-677c-4beb-bb20-53c4f7fcd60a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c598 bef21930-677c-4beb-bb20-53c4f7fcd60a /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095352887377922,"min-table-barrier-ts":450095352887377922,"admin-job-type":0} /tidb/cdc/default/default/task/position/bef21930-677c-4beb-bb20-53c4f7fcd60a/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0012b9e90 TaskPositions:map[bef21930-677c-4beb-bb20-53c4f7fcd60a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095352887377925, checkpointTs: 450095352887377922, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bef21930-677c-4beb-bb20-53c4f7fcd60a {"id":"bef21930-677c-4beb-bb20-53c4f7fcd60a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c598 bef21930-677c-4beb-bb20-53c4f7fcd60a /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095352887377922,"min-table-barrier-ts":450095352887377922,"admin-job-type":0} /tidb/cdc/default/default/task/position/bef21930-677c-4beb-bb20-53c4f7fcd60a/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0012b9e90 TaskPositions:map[bef21930-677c-4beb-bb20-53c4f7fcd60a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095352887377925, checkpointTs: 450095352887377922, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bef21930-677c-4beb-bb20-53c4f7fcd60a {"id":"bef21930-677c-4beb-bb20-53c4f7fcd60a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c598 bef21930-677c-4beb-bb20-53c4f7fcd60a /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095352887377922,"min-table-barrier-ts":450095352887377922,"admin-job-type":0} /tidb/cdc/default/default/task/position/bef21930-677c-4beb-bb20-53c4f7fcd60a/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:11:52.631 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.644 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.647 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.652 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.653 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:11:52.727 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:52.826 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.827 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:52.832 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.834 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.842 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:11:52.936 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:53.024 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/29 18:11:53.234 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/29 18:11:53.242 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.324 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:53.338 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/29 18:11:53.355 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:11:53 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2981929821.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:53.526 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.730 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:53.731 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.736 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:53.743 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:11:53.745 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:53.825 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:53.932 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/29 18:11:53.934 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:53.936 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.938 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/29 18:11:53.942 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.945 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:53.945 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:11:54.024 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.044 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/29 18:11:54.240 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.240 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] table generate_column.t not exists for 1-th check, retry later table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:54.325 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.339 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.447 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.645 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.649 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/29 18:11:54.650 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/29 18:11:54.651 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.731 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.731 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.737 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:11:54.751 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:54.940 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.946 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.946 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:54.946 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:54.952 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:11:55.022 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table generate_column.t1 exists check diff failed 1-th time, retry later [2024/05/29 18:11:55.038 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/29 18:11:55.324 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/29 18:11:55.326 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:55.332 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/29 18:11:55.339 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.424 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.525 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later [2024/05/29 18:11:55.736 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.737 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:55.747 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.751 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.826 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:11:55.842 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:55.949 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:56.033 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:56.033 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.033 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:11:56.034 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:56.037 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/29 18:11:56.039 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f554001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25708, start at 2024-05-29 18:11:23.140442537 +0800 CST m=+4.562226122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:11:56.052 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/29 18:11:56.322 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/29 18:11:56.338 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:56.341 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.424 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.525 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.finish_mark not exists for 2-th check, retry later [2024/05/29 18:11:56.726 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.726 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/29 18:11:56.730 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/29 18:11:56.743 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.746 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.748 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:11:56 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bcebd22-8ba9-41cc-8240-f42417f72a27 {"id":"2bcebd22-8ba9-41cc-8240-f42417f72a27","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d4b1 2bcebd22-8ba9-41cc-8240-f42417f72a27 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bcebd22-8ba9-41cc-8240-f42417f72a27 {"id":"2bcebd22-8ba9-41cc-8240-f42417f72a27","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d4b1 2bcebd22-8ba9-41cc-8240-f42417f72a27 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bcebd22-8ba9-41cc-8240-f42417f72a27 {"id":"2bcebd22-8ba9-41cc-8240-f42417f72a27","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/75d365ce-eb24-4edd-8714-599f730c6cf4 {"id":"75d365ce-eb24-4edd-8714-599f730c6cf4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d48e 75d365ce-eb24-4edd-8714-599f730c6cf4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d582d4b1 2bcebd22-8ba9-41cc-8240-f42417f72a27 /tidb/cdc/default/default/upstream/7374362016806282648 {"id":7374362016806282648,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583dc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25345, start at 2024-05-29 18:11:32.24366434 +0800 CST m=+4.259939321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:11:56.837 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:11:56.929 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.041 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.131 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:57.137 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/29 18:11:57.142 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:57.148 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:57.150 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:57.232 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:11:57.344 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/29 18:11:57.441 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.526 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.541 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:57.548 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/29 18:11:57.549 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:57.651 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:57.740 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.831 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.835 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.924 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:11:57.949 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:58.029 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.129 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.229 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/29 18:11:58.231 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:58.242 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:58.249 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/29 18:11:58.325 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:58.327 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.finish_mark not exists for 3-th check, retry later [2024/05/29 18:11:58.336 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:11:58.431 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/29 18:11:58.526 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.534 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.623 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:58.649 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:58.829 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.928 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.937 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:58.942 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:11:59.043 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/29 18:11:59.045 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:11:59.046 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:59.226 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:11:59.329 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:11:59 CST 2024] <<<<<< run test case generate_column success! >>>>>> [2024/05/29 18:11:59.425 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:59.437 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:59.445 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:59.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:11:59.539 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:11:59.627 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/29 18:11:59.640 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:11:59.641 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:11:59.821 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:11:59.836 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/29 18:11:59.839 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:11:59.842 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:11:59.942 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:00.038 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:00.047 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:00.130 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:00.149 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:00.224 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:00.250 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:00.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:00.444 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/29 18:12:00.445 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:00.623 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (23993) - No such process wait process 23993 exit for 1-th time... process 23993 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:11:59 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2420324205.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:00.625 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/29 18:12:00.627 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:00.630 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:00.645 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:00.650 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/29 18:12:00.728 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:00.729 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 4-th check, retry later [2024/05/29 18:12:00.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.022 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.048 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:01.138 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:01.236 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:01.240 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:01.340 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:01.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.443 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:01.522 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/29 18:12:01.639 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.641 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] check_redo_resolved_ts fd512f00-2883-40fa-9f5e-881b3b91c56c 450095355376173063 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/fd512f00-2883-40fa-9f5e-881b3b91c56c/meta global resolved ts 450095349609005077 not forward to 450095355376173063 run task failed 1-th time, retry later [2024/05/29 18:12:01.721 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.746 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:01.826 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:01.840 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:01.928 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/29 18:12:01.933 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:01.940 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:02.125 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.142 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/29 18:12:02.142 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.239 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:02.331 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:02.348 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc000a5aea0 TaskPositions:map[e15fa915-5d5c-4a33-ba3f-53b454e91312:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e15fa915-5d5c-4a33-ba3f-53b454e91312 {"id":"e15fa915-5d5c-4a33-ba3f-53b454e91312","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c5ea e15fa915-5d5c-4a33-ba3f-53b454e91312 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/e15fa915-5d5c-4a33-ba3f-53b454e91312/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc000a5aea0 TaskPositions:map[e15fa915-5d5c-4a33-ba3f-53b454e91312:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e15fa915-5d5c-4a33-ba3f-53b454e91312 {"id":"e15fa915-5d5c-4a33-ba3f-53b454e91312","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c5ea e15fa915-5d5c-4a33-ba3f-53b454e91312 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/e15fa915-5d5c-4a33-ba3f-53b454e91312/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc000a5aea0 TaskPositions:map[e15fa915-5d5c-4a33-ba3f-53b454e91312:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e15fa915-5d5c-4a33-ba3f-53b454e91312 {"id":"e15fa915-5d5c-4a33-ba3f-53b454e91312","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c5ea e15fa915-5d5c-4a33-ba3f-53b454e91312 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,E+ grep -q 'etcd info' RROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/e15fa915-5d5c-4a33-ba3f-53b454e91312/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/29 18:12:02.425 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:02.529 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:02.542 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.633 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table ddl_attributes.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 5-th check, retry later [2024/05/29 18:12:02.750 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.837 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.839 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/29 18:12:02.934 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:02.942 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.030 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:03.047 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:03.123 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:03.233 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.332 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.430 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] check_redo_resolved_ts fd512f00-2883-40fa-9f5e-881b3b91c56c 450095355376173063 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/fd512f00-2883-40fa-9f5e-881b3b91c56c/meta [2024/05/29 18:12:03.527 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:03.537 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:03.541 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:03.647 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f23c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28564, start at 2024-05-29 18:11:22.931715412 +0800 CST m=+4.467770634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:22.939 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:22.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:22.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... global resolved ts 450095353528057858 not forward to 450095355376173063 run task failed 2-th time, retry later [2024/05/29 18:12:03.726 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/29 18:12:03.727 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/29 18:12:03.743 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:03.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:03.940 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/29 18:12:04.029 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:04.047 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:04.126 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:04.128 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:04.141 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:04.324 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:04.346 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:04.425 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:04.530 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:04.543 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:04.546 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] table test.finish_mark not exists for 6-th check, retry later [2024/05/29 18:12:04.742 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:04.751 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:04.930 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/29 18:12:04.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:04.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:05.037 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:05.038 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/29 18:12:05.123 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:05.130 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table ddl_attributes.finish_mark exists check diff successfully [2024/05/29 18:12:05.335 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:12:05 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> [2024/05/29 18:12:05.522 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:05.534 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:05.540 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:05.656 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:05.748 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/29 18:12:05.755 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:05.831 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:05.848 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:06.038 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:06.137 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/29 18:12:06.138 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/29 18:12:06.144 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:06.157 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:06.335 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/29 18:12:06.340 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:06.351 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:06.368 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:06.472 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] table test.finish_mark not exists for 7-th check, retry later [2024/05/29 18:12:06.562 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/29 18:12:06.725 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/29 18:12:06.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/29 18:12:06.752 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:06.882 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/29 18:12:07.026 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse7f59fc4_d004_4b0b_b250_22a3d0c1311f"] [2024/05/29 18:12:07.035 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa171a64b_dc87_4e46_9046_e6fedce557ab"] [2024/05/29 18:12:07.051 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd9922633_bb38_441e_ac17_c874cce446f1"] [2024/05/29 18:12:07.148 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6bfd0010_075f_46c5_bf2a_54c830d8a408"] [2024/05/29 18:12:07.163 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/29 18:12:04.295 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/29 18:12:04.533 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/29 18:12:04.545 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:04.751 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:12:04.753 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/29 18:12:04.755 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:12:04.757 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:12:06.000 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/29 18:12:06.002 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:12:06.225 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:12:06.227 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/29 18:12:06.234 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:12:06.236 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:12:06.437 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/29 18:12:06.447 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/29 18:12:07.441 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa1d7e115_c904_4c3b_8549_7040ef9fdb2e"] [2024/05/29 18:12:07.444 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs077acb75_ef4e_4048_acfd_234f990a09da"] [2024/05/29 18:12:07.655 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/29 18:12:07.661 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:12:07.661 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/29 18:12:07.667 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:12:07.540 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/29 18:12:07.546 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/29 18:12:07.731 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs044ccaaf_55af_4db9_8dfd_f3c051244d5b"] [2024/05/29 18:12:07.742 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:07.751 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:07.830 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/29 18:12:07.851 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/29 18:12:08.006 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:12:08.011 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] check_redo_resolved_ts fd512f00-2883-40fa-9f5e-881b3b91c56c 450095355376173063 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/fd512f00-2883-40fa-9f5e-881b3b91c56c/meta run task successfully [2024/05/29 18:12:07.835 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:07.927 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:07.939 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/29 18:12:07.953 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:08.030 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:08.032 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:08.034 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs96467637_c642_4f31_af57_c961aabfba60"] [2024/05/29 18:12:08.036 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:12:08.042 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:12:08.132 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:08.246 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:08.433 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:08.434 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:08.526 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table test.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/29 18:12:08.739 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:08.827 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:08.844 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:08.849 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:08.926 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:09.030 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/29 18:12:09.032 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/05/29 18:12:09.042 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] wait process cdc.test exit for 2-th time... [2024/05/29 18:12:09.125 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:09.137 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:09.149 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:09.222 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:09.233 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:09.243 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:12:08 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/05/29 18:12:09.352 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:09.525 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:09.528 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:09.478 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:12:09.478 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:12:09.479 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095349766553612] [resolvedTs=450095356674048004] [2024/05/29 18:12:09.479 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:12:09.479 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:12:09.485 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=6.219436ms] [2024/05/29 18:12:09.490 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:12:09.494 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.495 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:12:09.496 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:12:09.496 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:12:09.496 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:12:09.501 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:12:09.504 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:12:09.832 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:10.125 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=646.327365ms] [2024/05/29 18:12:10.125 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":450095349766553606,\"CommitTs\":450095349766553612,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_file\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/29 18:12:10.325 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:10.329 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:10.537 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:10.636 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:10.636 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:10.836 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish_mark not exists for 9-th check, retry later [2024/05/29 18:12:11.028 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:11.127 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:11.141 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.077 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095356674048004] [2024/05/29 18:12:11.078 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095356674048004] [2024/05/29 18:12:11.178 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095356674048004] [2024/05/29 18:12:11.178 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095356674048004] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:11.229 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.245 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:11.248 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.346 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:11.355 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.355 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:11.445 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.631 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:11.641 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:11.730 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:11.750 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:11.751 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:11.845 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:12.082 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450095356674048004] [2024/05/29 18:12:12.082 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450095356674048004] [2024/05/29 18:12:11.935 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:12.023 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:12.136 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:12.183 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=450095356674048004] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:12:12.183 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully [2024/05/29 18:12:12.234 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:12.335 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24208) - No such process wait process 24208 exit for 1-th time... process 24208 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:12:10 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2439724399.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:12.434 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:12.549 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:12.550 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:12.624 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:12.650 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:12.832 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:12.844 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff successfully [Wed May 29 18:12:12 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 table test.finish_mark not exists for 10-th check, retry later [2024/05/29 18:12:12.938 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:12.951 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:13.031 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:13.044 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.132 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:13.233 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:13.329 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.340 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.441 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:13.539 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.634 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:13.638 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:13.646 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:13.746 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.851 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:13.936 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:14.035 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:14.038 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:14.053 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:14.126 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:14.140 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:14.145 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc00163ef78 TaskPositions:map[daac3952-4154-424c-acd7-3f3a9928d621:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daac3952-4154-424c-acd7-3f3a9928d621 {"id":"daac3952-4154-424c-acd7-3f3a9928d621","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c61d daac3952-4154-424c-acd7-3f3a9928d621 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/daac3952-4154-424c-acd7-3f3a9928d621/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc00163ef78 TaskPositions:map[daac3952-4154-424c-acd7-3f3a9928d621:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daac3952-4154-424c-acd7-3f3a9928d621 {"id":"daac3952-4154-424c-acd7-3f3a9928d621","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c61d daac3952-4154-424c-acd7-3f3a9928d621 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,E+ grep -q 'failed to get info:' RROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/daac3952-4154-424c-acd7-3f3a9928d621/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc00163ef78 TaskPositions:map[daac3952-4154-424c-acd7-3f3a9928d621:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daac3952-4154-424c-acd7-3f3a9928d621 {"id":"daac3952-4154-424c-acd7-3f3a9928d621","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c61d daac3952-4154-424c-acd7-3f3a9928d621 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/daac3952-4154-424c-acd7-3f3a9928d621/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/29 18:12:14.332 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:14.428 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:14.435 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 25.28604935s"] [2024/05/29 18:12:14.527 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:14.637 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.finish_mark not exists for 11-th check, retry later [2024/05/29 18:12:14.842 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/29 18:12:14.739 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:14.744 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:14.746 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:14.934 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:15.074 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:15.078 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f58c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25607, start at 2024-05-29 18:11:23.14385169 +0800 CST m=+4.630298452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f554001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:25708, start at 2024-05-29 18:11:23.140442537 +0800 CST m=+4.562226122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:12:14 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.2970829710.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:15.040 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.043 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:15.143 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:15.152 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.286 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:12:15.288 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:12:15.290 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/29 18:12:15.291 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:12:15.225 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.235 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.245 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:15.256 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:15.348 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:15.432 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:15.444 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:12:15.491 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/29 18:12:15.593 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:12:15.714 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:12:15.715 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:12:15.805 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:12:15.807 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [2024/05/29 18:12:15.734 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:15.740 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.745 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:15.936 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:16.036 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:15.932 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/29 18:12:16.003 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 [2024/05/29 18:12:16.137 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.148 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:16.237 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:16.142 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:12:16.143 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/29 18:12:16.203 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/29 18:12:16.204 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:12:16.362 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/29 18:12:16.330 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.330 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.341 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.351 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:16.526 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:16.408 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/29 18:12:16.571 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:12:16.572 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:12:16.602 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:12:16.610 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583e8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25270, start at 2024-05-29 18:11:32.265470749 +0800 CST m=+4.356135410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5583dc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:25345, start at 2024-05-29 18:11:32.24366434 +0800 CST m=+4.259939321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:32.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:32.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:32.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.29356.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 12-th check, retry later [2024/05/29 18:12:16.547 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:16.626 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:16.631 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:16.729 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.797 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/29 18:12:16.833 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] + set +x + tso='450095359309905921 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095359309905921 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:12:16 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.2939729399.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:16.824 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:16.836 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) [2024/05/29 18:12:17.017 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/29 18:12:17.021 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/29 18:12:17.071 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/29 18:12:17.079 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/29 18:12:17.134 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:17.246 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:17.248 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Collecting certifi>=2017.4.17 (from requests==2.26.0) [2024/05/29 18:12:17.333 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/29 18:12:17.384 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/29 18:12:17.424 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:17.438 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:17.452 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:17.539 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) [2024/05/29 18:12:17.544 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/29 18:12:17.549 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/29 18:12:17.599 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/29 18:12:17.610 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/29 18:12:17.624 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:17.643 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) [2024/05/29 18:12:17.754 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/29 18:12:17.813 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:17.828 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:17.932 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.023 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:18.027 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.029 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) [2024/05/29 18:12:17.963 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/29 18:12:17.966 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/29 18:12:18.025 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/29 18:12:18.039 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/29 18:12:18.131 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:18.241 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.167 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/29 18:12:18.246 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/29 18:12:18.377 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/29 18:12:18.381 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:17 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ad9f99c-2572-4534-9c87-0d7cc642dc6f {"id":"3ad9f99c-2572-4534-9c87-0d7cc642dc6f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2de8e 3ad9f99c-2572-4534-9c87-0d7cc642dc6f /tidb/cdc/default/default/upstream/7374362098489403720 {"id":7374362098489403720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ad9f99c-2572-4534-9c87-0d7cc642dc6f {"id":"3ad9f99c-2572-4534-9c87-0d7cc642dc6f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2de8e 3ad9f99c-2572-4534-9c87-0d7cc642dc6f /tidb/cdc/default/default/upstream/7374362098489403720 {"id":7374362098489403720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ad9f99c-2572-4534-9c87-0d7cc642dc6f {"id":"3ad9f99c-2572-4534-9c87-0d7cc642dc6f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2de8e 3ad9f99c-2572-4534-9c87-0d7cc642dc6f /tidb/cdc/default/default/upstream/7374362098489403720 {"id":7374362098489403720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/29 18:12:18.422 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.445 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:18.534 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.536 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 13-th check, retry later Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) [2024/05/29 18:12:18.465 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/29 18:12:18.480 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/29 18:12:18.589 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/29 18:12:18.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:18.634 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:18.721 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:18.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:18.736 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) [2024/05/29 18:12:18.673 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/29 18:12:18.793 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/29 18:12:18.796 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/29 18:12:18.888 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/29 18:12:18.901 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/29 18:12:18.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:18.941 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.033 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) Installing collected packages: certifi, urllib3, idna, requests Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 [2024/05/29 18:12:19.001 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/29 18:12:19.112 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:19.134 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.142 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:19.144 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 [2024/05/29 18:12:19.223 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/29 18:12:19.227 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/29 18:12:19.321 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/29 18:12:19.335 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/29 18:12:19.418 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:19.440 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.452 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:19.549 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/29 18:12:19.617 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/29 18:12:19.621 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/29 18:12:19.623 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:19.642 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.646 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:19.734 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:19.746 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.750 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:19.833 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:19.766 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/29 18:12:19.780 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/29 18:12:19.836 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] The 1 times to try to start tidb cluster... [2024/05/29 18:12:19.857 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:19.925 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:19.945 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:19 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c723ed80-1e76-4579-8cd2-47c766db2cc4 {"id":"c723ed80-1e76-4579-8cd2-47c766db2cc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2dd88 c723ed80-1e76-4579-8cd2-47c766db2cc4 /tidb/cdc/default/default/upstream/7374362143666856731 {"id":7374362143666856731,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c723ed80-1e76-4579-8cd2-47c766db2cc4 {"id":"c723ed80-1e76-4579-8cd2-47c766db2cc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2dd88 c723ed80-1e76-4579-8cd2-47c766db2cc4 /tidb/cdc/default/default/upstream/7374362143666856731 {"id":7374362143666856731,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c723ed80-1e76-4579-8cd2-47c766db2cc4 {"id":"c723ed80-1e76-4579-8cd2-47c766db2cc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5e2dd88 c723ed80-1e76-4579-8cd2-47c766db2cc4 /tidb/cdc/default/default/upstream/7374362143666856731 {"id":7374362143666856731,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.29464.out cli changefeed create --start-ts=450095359309905921 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: c62268b0-6417-45b0-b5e0-f6aeb022127b Info: {"upstream_id":7374362143666856731,"namespace":"default","id":"c62268b0-6417-45b0-b5e0-f6aeb022127b","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-29T18:12:20.141492311+08:00","start_ts":450095359309905921,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095359309905921,"checkpoint_ts":450095359309905921,"checkpoint_time":"2024-05-29 18:12:16.430"} PASS [2024/05/29 18:12:20.132 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:20.342 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [2024/05/29 18:12:20.434 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:20.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:20.543 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:20.633 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:20.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:20.749 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:20.841 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:20.846 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:20.925 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:21.022 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:21.030 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:21.049 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table test.finish_mark not exists for 14-th check, retry later Verifying downstream PD is started... [2024/05/29 18:12:21.130 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:21.139 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:21.335 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f49c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28493, start at 2024-05-29 18:11:23.048398846 +0800 CST m=+4.647137849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:23.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:23.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:23.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f54f23c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:28564, start at 2024-05-29 18:11:22.931715412 +0800 CST m=+4.467770634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:13:22.939 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:11:22.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:01:22.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:21.529 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:12:21 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3239232394.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:21.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:21.638 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:21.729 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:21.833 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:21.844 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:21.941 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.030 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.032 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.034 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:22.051 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:22.133 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.142 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:22.151 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.243 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:22.628 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:22.732 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:22.741 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:22.825 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:22.829 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:22.831 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:12:22.933 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:22.955 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:22.956 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:23.026 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:23.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:23.132 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:23.325 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:23.334 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.finish_mark not exists for 15-th check, retry later [2024/05/29 18:12:23.526 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:23.535 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:23.749 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:23.836 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:23.932 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:23.936 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:23.948 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:23.956 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:24.031 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:24.054 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:24.133 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:24.139 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24402) - No such process wait process 24402 exit for 1-th time... process 24402 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:12:22 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2458524587.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:24.325 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:24.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:24.529 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:24.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:24.800 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:24.851 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsce3c675d_ae12_4d6a_a976_d1e1a5975765"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9ba1cd7-d71f-4859-93a0-925b50ebf07b {"id":"c9ba1cd7-d71f-4859-93a0-925b50ebf07b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5feff8e c9ba1cd7-d71f-4859-93a0-925b50ebf07b /tidb/cdc/default/default/upstream/7374362104350516122 {"id":7374362104350516122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9ba1cd7-d71f-4859-93a0-925b50ebf07b {"id":"c9ba1cd7-d71f-4859-93a0-925b50ebf07b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5feff8e c9ba1cd7-d71f-4859-93a0-925b50ebf07b /tidb/cdc/default/default/upstream/7374362104350516122 {"id":7374362104350516122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9ba1cd7-d71f-4859-93a0-925b50ebf07b {"id":"c9ba1cd7-d71f-4859-93a0-925b50ebf07b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d5feff8e c9ba1cd7-d71f-4859-93a0-925b50ebf07b /tidb/cdc/default/default/upstream/7374362104350516122 {"id":7374362104350516122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 64edda7d-03e4-4a68-88a2-7b66c1bd512e Info: {"upstream_id":7374362104350516122,"namespace":"default","id":"64edda7d-03e4-4a68-88a2-7b66c1bd512e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:12:24.814296798+08:00","start_ts":450095360647888897,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095360647888897,"checkpoint_ts":450095360647888897,"checkpoint_time":"2024-05-29 18:12:21.534"} table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later [2024/05/29 18:12:25.043 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1d17c241_67b2_411a_a783_07a714958d14"] [2024/05/29 18:12:25.135 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs557a6637_48d2_4fa3_abb8_aa84ae73a2ac"] [2024/05/29 18:12:25.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:25.342 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb87c325e_eaf1_4861_94c6_46dc04010a72"] [2024/05/29 18:12:25.366 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:25.428 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2284b20c_af0a_47fe_b18a_8318ea563dd2"] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.finish_mark not exists for 16-th check, retry later [2024/05/29 18:12:25.452 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:25.533 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5a27f37d_ad1a_46e0_9923_7db8150eb9b0"] [2024/05/29 18:12:25.550 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:25.622 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:25.645 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse459bc72_b4b9_4a6d_b5cd_e3470ed85ded"] [2024/05/29 18:12:25.657 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:25.732 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:26.032 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:26.135 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:26.136 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0018e31e8 TaskPositions:map[30ba2e7e-43b7-4fce-875f-5dd28af3c1f1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 {"id":"30ba2e7e-43b7-4fce-875f-5dd28af3c1f1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c65d 30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0018e31e8 TaskPositions:map[30ba2e7e-43b7-4fce-875f-5dd28af3c1f1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 {"id":"30ba2e7e-43b7-4fce-875f-5dd28af3c1f1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c65d 30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc0018e31e8 TaskPositions:map[30ba2e7e-43b7-4fce-875f-5dd28af3c1f1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 {"id":"30ba2e7e-43b7-4fce-875f-5dd28af3c1f1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c65d 30ba2e7e-43b7-4fce-875f-5dd28af3c1f1 /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353018187785,"admin-job-type":0} /tidb/cdc/default/default/task/position/30ba2e7e-43b7-4fce-875f-5dd28af3c1f1/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/29 18:12:26.424 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfe7e7203_ac40_4310_9f89_6fa87983f59e"] [2024/05/29 18:12:26.426 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:26.431 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:26.441 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:26.449 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:25.766 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/29 18:12:26.017 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:26.017 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/29 18:12:26.222 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:12:26.224 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:12:26.230 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/29 18:12:26.231 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:12:26.455 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:12:26.458 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/29 18:12:26.532 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:26.636 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:26.639 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:26.648 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:26.665 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:12:26.667 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/29 18:12:26.669 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:12:26.670 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:12:26.723 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:26.732 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:26.748 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:26.927 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:26.866 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/29 18:12:26.869 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] check diff failed 2-th time, retry later [2024/05/29 18:12:27.024 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:27.028 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:27.070 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/29 18:12:27.072 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:12:27.076 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:12:27.078 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.finish_mark exists [2024/05/29 18:12:27.235 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:27.325 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:27.328 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:27.333 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:27.333 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:27.432 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:27.403 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/29 18:12:27.414 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff successfully [2024/05/29 18:12:27.527 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:27.641 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:27.642 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:27.643 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:27.724 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:27.594 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:12:27.596 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/29 18:12:27.604 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:12:27.605 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:12:27.738 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:27.748 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:27.928 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:27.944 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:27.949 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:27.794 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/29 18:12:27.811 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/29 18:12:27.983 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/29 18:12:27.985 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/29 18:12:27.998 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/29 18:12:27.999 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:12:28 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/29 18:12:28.050 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:28.054 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:28.122 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:28.126 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:28.129 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:28.225 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:28.188 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/29 18:12:28.209 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/29 18:12:28.341 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:28.394 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/29 18:12:28.397 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/29 18:12:28.409 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/29 18:12:28.410 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/29 18:12:28.624 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:28.631 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:28.637 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:28.642 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:28.739 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:28.585 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/29 18:12:28.597 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/29 18:12:28.781 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/29 18:12:28.783 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] check diff successfully ERROR 1146 (42S02) at line 1: Table 'sequence_test.seq0' doesn't exist [2024/05/29 18:12:28.747 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:28.847 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:28.939 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:28.946 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:28.796 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/29 18:12:28.797 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/29 18:12:28.981 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/29 18:12:28.995 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/29 18:12:29.030 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:29.033 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:29.034 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:29.123 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:29.126 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:29.140 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:29.143 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [2024/05/29 18:12:29.273 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/29 18:12:29.275 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/29 18:12:29.291 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/29 18:12:29.292 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/29 18:12:29.350 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:29.442 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:29.471 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/29 18:12:29.491 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:29.539 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:29.625 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:29.731 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:29.744 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:29.659 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/29 18:12:29.661 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/29 18:12:29.690 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/29 18:12:29.691 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] table sequence_test.mark_table not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e938001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25122, start at 2024-05-29 18:12:27.900108616 +0800 CST m=+4.380669690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/29 18:12:29.831 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:29.853 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:29.941 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:29.943 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:29.944 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:29.946 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:29.855 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/29 18:12:29.891 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/29 18:12:30.054 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/29 18:12:30.056 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log [2024/05/29 18:12:30.046 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:30.050 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:30.055 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:30.135 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:30.233 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:30.077 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/29 18:12:30.078 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/29 18:12:30.252 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/29 18:12:30.275 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/29 18:12:30.322 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:30.349 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:30.352 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:30.446 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:30.445 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/29 18:12:30.447 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/29 18:12:30.464 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/29 18:12:30.465 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/29 18:12:30.628 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:30.635 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:30.743 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:30.631 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/29 18:12:30.651 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/29 18:12:30.840 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:30.843 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:30.844 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:30.847 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:30.943 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:30.946 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:30.949 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:31.036 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:31.126 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:31.151 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:31.326 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:31.327 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:31.336 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:31.433 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:31.440 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:31.637 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:31.824 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:31.828 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:31.832 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:31.833 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:32.024 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:32.028 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:32.031 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:32.047 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:32.131 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:32.228 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table sequence_test.mark_table exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:12:32 CST 2024] <<<<<< run test case sequence success! >>>>>> [2024/05/29 18:12:32.330 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:32.335 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:32.339 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:32.348 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:32.434 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:32.453 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:32.540 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:32.544 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:32.551 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Starting CDC state checker... [2024/05/29 18:12:32.633 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table owner_remove_table_error.finished_mark not exists for 1-th check, retry later table owner_remove_table_error.finished_mark not exists for 2-th check, retry later table owner_remove_table_error.finished_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:12:27 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> [2024/05/29 18:12:32.828 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:32.923 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:32.926 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:32.944 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:33.024 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:33.126 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:33.238 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:33.247 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:12:33.326 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:33.326 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:33.348 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:33.445 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:33.524 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:33.537 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:33.541 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:33.542 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:33.634 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:33.643 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:33.644 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:33.728 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:33.838 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:34.134 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:34.136 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:34.139 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:34.237 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:34.331 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:34.337 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:34.440 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:34.451 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:34.625 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:34.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:34.638 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:34.638 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:34.643 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:34.735 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:34.832 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:34.944 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:34.950 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:34.950 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:35.048 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:35.048 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:35.143 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:35.240 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:35.246 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:35.347 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:35.351 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:35.353 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:35.430 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:35.444 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:35.444 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:35.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5943900017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24777, start at 2024-05-29 18:12:33.669357471 +0800 CST m=+4.395852463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:33.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:33.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:33.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log [2024/05/29 18:12:35.624 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:35.633 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:35.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:35.842 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:35.935 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:35.936 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:36.026 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24590) - No such process wait process 24590 exit for 1-th time... process 24590 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:12:34 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2477624778.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:12:36.122 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:36.150 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:36.239 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:36.244 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:36.255 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:36.331 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:36.344 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:36.346 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:36.353 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:36.438 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:36.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:36.445 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:36.824 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:36.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:36.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:36.929 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:37.043 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:12:36.670 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/29 18:12:36.885 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/29 18:12:36.903 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:37.061 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:12:37.064 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/29 18:12:37.112 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:12:37.114 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:12:37.277 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/29 18:12:37.308 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:12:37.240 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:37.249 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:37.334 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:37.478 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:12:37.480 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:12:37.492 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:12:37.493 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:37 GMT [2024/05/29 18:12:37.435 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] < Content-Type: text/plain; charset=utf-8 [2024/05/29 18:12:37.440 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] < Transfer-Encoding: chunked < { [data not shown] [2024/05/29 18:12:37.442 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] * Connection #0 to host 127.0.0.1 left intact + res=' [2024/05/29 18:12:37.451 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] *** owner info ***: [2024/05/29 18:12:37.624 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:37.630 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] *** processors info ***: [2024/05/29 18:12:37.642 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc002cba918 TaskPositions:map[8fce044d-0a25-41c2-83a9-689bb4f9f0de:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing tableID: 84, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fce044d-0a25-41c2-83a9-689bb4f9f0de {"id":"8fce044d-0a25-41c2-83a9-689bb4f9f0de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c694 8fce044d-0a25-41c2-83a9-689bb4f9f0de /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353149259814,"admin-job-type":0} /tidb/cdc/default/default/task/position/8fce044d-0a25-41c2-83a9-689bb4f9f0de/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc002cba918 TaskPositions:map[8fce044d-0a25-41c2-83a9-689bb4f9f0de:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing tableID: 84, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fce044d-0a25-41c2-83a9-689bb4f9f0de {"id":"8fce044d-0a25-41c2-83a9-689bb4f9f0de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c694 8fce044d-0a25-41c2-83a9-689bb4f9f0de /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353149259814,"admin-job-type":0} /tidb/cdc/default/default/task/position/8fce044d-0a25-41c2-83a9-689bb4f9f0de/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {ClusterID:default ID:default/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be Info:{"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} Status:0xc002cba918 TaskPositions:map[8fce044d-0a25-41c2-83a9-689bb4f9f0de:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing tableID: 84, resolvedTs: 450095353018187785, checkpointTs: 450095353018187785, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fce044d-0a25-41c2-83a9-689bb4f9f0de {"id":"8fce044d-0a25-41c2-83a9-689bb4f9f0de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d490c694 8fce044d-0a25-41c2-83a9-689bb4f9f0de /tidb/cdc/default/default/changefeed/info/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"upstream-id":7374361327621511023,"namespace":"default","changefeed-id":"15833a8e-6ee7-4ba9-9c3f-bcefaefae6be","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:10:50.971667859+08:00","start-ts":450095336870379526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095336896593926} /tidb/cdc/default/default/changefeed/status/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":450095353018187785,"min-table-barrier-ts":450095353149259814,"admin-job-type":0} /tidb/cdc/default/default/task/position/8fce044d-0a25-41c2-83a9-689bb4f9f0de/15833a8e-6ee7-4ba9-9c3f-bcefaefae6be {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361327621511023 + grep -q 'etcd info' {"id":7374361327621511023,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 Starting CDC state checker... [2024/05/29 18:12:37.655 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/29 18:12:37.664 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/29 18:12:37.840 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/29 18:12:37.842 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:12:37.845 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/29 18:12:37.847 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:12:37.746 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later [2024/05/29 18:12:38.032 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/29 18:12:38.040 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596f0c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25437, start at 2024-05-29 18:12:36.427976226 +0800 CST m=+4.813037569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log [2024/05/29 18:12:38.021 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:38.035 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:38.037 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:38.161 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:38.217 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:12:38.219 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/29 18:12:38.239 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:12:38.242 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:12:38.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:38.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:38.237 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:38.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:38.353 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:38.412 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/29 18:12:38.441 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/29 18:12:38.587 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/29 18:12:38.589 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/29 18:12:38.617 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/29 18:12:38.619 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/29 18:12:38.441 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:38.540 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:38.565 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:38.667 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:38.675 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:38.747 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/29 18:12:38.794 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/29 18:12:38.731 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:38.748 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:38.835 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:38.920 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/29 18:12:38.922 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/29 18:12:38.970 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/29 18:12:38.972 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/29 18:12:39.095 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/29 18:12:39.051 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:39.149 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/29 18:12:39.270 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/29 18:12:39.272 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/29 18:12:39.336 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/29 18:12:39.338 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/29 18:12:39.235 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:39.250 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb4ad8659_aaaa_4ecb_af28_78098443f58d"] [2024/05/29 18:12:39.281 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf3e4b94b_98f1_449d_9bdc_d27808856825"] [2024/05/29 18:12:39.329 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs28e15c5d_34b8_484b_b6b1_4e389066a686"] [2024/05/29 18:12:39.338 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:39.359 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs12566461_5891_4f55_82d2_0ed07936847f"] [2024/05/29 18:12:39.366 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf2b59e81_8ac9_4f4d_b6e9_97f9a31f8832"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:39.624 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb69b9103_88a4_4df8_acd3_cc8df20fef48"] table kill_owner_with_ddl.t3 exists [2024/05/29 18:12:39.725 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4d6d8ea8_11dc_4b92_9557_067a8735e3ab"] [2024/05/29 18:12:39.936 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:39.938 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Starting CDC state checker... check diff failed 1-th time, retry later [2024/05/29 18:12:40.039 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.054 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:40.127 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.142 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later [2024/05/29 18:12:40.234 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.240 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.244 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:40.247 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:40.435 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:40.442 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.524 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:40.545 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:40.645 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:40.430 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] table processor_resolved_ts_fallback.t1 exists [Wed May 29 18:12:40 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2234222344.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:12:40.735 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:40.923 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:40.934 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.030 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:41.038 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.147 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.226 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:41.115 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:12:41.228 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.234 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:41.350 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.435 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:41.440 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:41.451 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4ab8a414_12a7_4a04_a546_fe91a3410a39"] [2024/05/29 18:12:41.536 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:41.545 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:41.734 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:41.747 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:41.836 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:41.838 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:41.933 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:41.958 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:42.030 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:42.040 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:42.054 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:42.150 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:42.240 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:12:42.251 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:42.330 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:42.331 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:42.332 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:42.349 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:42.445 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:12:42 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> [2024/05/29 18:12:42.523 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:42.545 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:42.545 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:42.740 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:42.923 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:42.932 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:42.947 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:42.957 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:43.051 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:43.143 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:43.229 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:43.244 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:43.326 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:43.334 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:43.351 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:43.430 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:43.445 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:43.446 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:43.533 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:43.639 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:43.732 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:43.824 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:12:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/b8581dba-e847-4a1f-b10d-269632b5db2a {ClusterID:default ID:default/b8581dba-e847-4a1f-b10d-269632b5db2a Info:{"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} Status:0xc002e9a9c0 TaskPositions:map[3e009730-9239-425b-bc0b-0fd32582c3d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 {"id":"dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f390 dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 /tidb/cdc/default/default/changefeed/info/b8581dba-e847-4a1f-b10d-269632b5db2a {"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} /tidb/cdc/default/default/changefeed/status/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":450095366256459785,"min-table-barrier-ts":450095366256459785,"admin-job-type":0} /tidb/cdc/default/default/task/position/3e009730-9239-425b-bc0b-0fd32582c3d1/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/b8581dba-e847-4a1f-b10d-269632b5db2a {ClusterID:default ID:default/b8581dba-e847-4a1f-b10d-269632b5db2a Info:{"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} Status:0xc002e9a9c0 TaskPositions:map[3e009730-9239-425b-bc0b-0fd32582c3d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 {"id":"dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f390 dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 /tidb/cdc/default/default/changefeed/info/b8581dba-e847-4a1f-b10d-269632b5db2a {"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} /tidb/cdc/default/default/changefeed/status/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":450095366256459785,"min-table-barrier-ts":450095366256459785,"admin-job-type":0} /tidb/cdc/default/default/task/position/3e009730-9239-425b-bc0b-0fd32582c3d1/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/b8581dba-e847-4a1f-b10d-269632b5db2a {ClusterID:default ID:default/b8581dba-e847-4a1f-b10d-269632b5db2a Info:{"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} Status:0xc002e9a9c0 TaskPositions:map[3e009730-9239-425b-bc0b-0fd32582c3d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e009730-9239-425b-bc0b-0fd32582c3d1 {"id":"3e009730-9239-425b-bc0b-0fd32582c3d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 {"id":"dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f288 3e009730-9239-425b-bc0b-0fd32582c3d1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d559f390 dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5 /tidb/cdc/default/default/changefeed/info/b8581dba-e847-4a1f-b10d-269632b5db2a {"upstream-id":7374361490244667338,"namespace":"default","changefeed-id":"b8581dba-e847-4a1f-b10d-269632b5db2a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:11:43.633374076+08:00","start-ts":450095350685106181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use+ grep -q 'etcd info' -file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095350711582721} /tidb/cdc/default/default/changefeed/status/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":450095366256459785,"min-table-barrier-ts":450095366256459785,"admin-job-type":0} /tidb/cdc/default/default/task/position/3e009730-9239-425b-bc0b-0fd32582c3d1/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd106b6c-6c5a-4d56-a6e0-8e2b68ee4dc5/b8581dba-e847-4a1f-b10d-269632b5db2a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374361490244667338 {"id":7374361490244667338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later [2024/05/29 18:12:43.945 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:44.022 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:44.024 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:44.045 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:44.124 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:44.144 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:44.324 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:44.334 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:44.430 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:44.437 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:44.450 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:44.534 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:44.624 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:44.640 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:44.644 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:44.729 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:44.927 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:44.928 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:44.933 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:44.944 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:45.021 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:45.045 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:45.227 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:45.235 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:45.241 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.251 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.324 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:45.351 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:45.426 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:45.430 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.442 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:45.444 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.646 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.655 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:45.726 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:45.727 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:45.735 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:45.835 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:46.031 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:46.032 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:46.036 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:46.124 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:46.137 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:46.226 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:46.247 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:46.329 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:46.329 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:46.339 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:46.537 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:46.542 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:46.552 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:46.624 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:46.630 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:46.735 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:46.947 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:46.950 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:47.024 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:47.031 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:47.050 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:47.146 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:47.235 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:47.238 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:47.246 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:47.254 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:47.434 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:47.439 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:47.443 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:47.529 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:47.534 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:47.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.022 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.029 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:48.030 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:48.035 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.046 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:48.136 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:48.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.239 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.242 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.252 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.346 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.349 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:48.350 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 22118 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later [2024/05/29 18:12:48.424 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.425 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.536 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:48.827 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:48.835 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:48.836 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.841 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:48.930 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:48.951 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:49.031 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:49.051 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:49.129 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:49.150 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:48.654 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/29 18:12:48.854 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:48.861 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/29 18:12:49.027 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:12:49.029 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:12:49.032 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/29 18:12:49.033 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:12:49.208 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:12:49.214 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/29 18:12:49.238 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:49.242 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:49.242 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:49.251 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:49.332 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:49.411 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:12:49.413 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/29 18:12:49.423 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:12:49.424 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:12:49.577 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/29 18:12:49.590 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] Verifying downstream PD is started... [2024/05/29 18:12:49.625 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:49.647 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:49.747 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/29 18:12:49.749 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:12:49.765 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:12:49.766 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/29 18:12:49.746 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:49.748 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:49.828 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:49.830 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:49.853 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:49.932 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:49.950 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/29 18:12:49.967 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/29 18:12:50.038 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:50.132 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:50.132 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:50.142 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' [2024/05/29 18:12:50.181 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:12:50.184 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/29 18:12:50.202 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:12:50.203 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:12:50.337 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:50.343 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later [2024/05/29 18:12:50.396 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/29 18:12:50.416 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/29 18:12:50.585 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/29 18:12:50.587 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/29 18:12:50.615 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/29 18:12:50.616 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:50.784 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/29 18:12:50.813 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:50.727 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:50.826 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:50.844 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:50.844 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:50.928 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:50.932 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:50.948 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:50.951 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:50.978 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/29 18:12:50.980 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/29 18:12:51.009 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/29 18:12:51.010 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/29 18:12:51.024 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:51.035 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:51.043 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:51.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:51.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:51.131 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a38340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26154, start at 2024-05-29 18:12:49.329124295 +0800 CST m=+4.397083952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log [2024/05/29 18:12:51.179 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/29 18:12:51.214 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/29 18:12:51.392 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/29 18:12:51.393 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:51.450 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:51.431 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/29 18:12:51.433 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/29 18:12:51.522 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:51.524 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:51.548 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:51.558 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:51.634 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:51.652 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:51.655 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/29 18:12:51.659 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:51.722 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:51.724 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:51.787 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/29 18:12:51.834 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/29 18:12:51.931 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5ef0e81e_07ab_4932_9342_5c03f6893dfa"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:52.021 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/29 18:12:52.023 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/29 18:12:52.063 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/29 18:12:52.064 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/29 18:12:52.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:52.030 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:52.050 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/29 18:12:52.228 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:52.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:52.237 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/29 18:12:52.277 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/29 18:12:52.322 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/29 18:12:52.436 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:52.439 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:52.523 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:52.527 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:12:52.653 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/29 18:12:52.655 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/29 18:12:52.549 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:52.738 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:52.738 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:52.748 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:52.701 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/29 18:12:52.702 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/29 18:12:52.882 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/29 18:12:52.925 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later [2024/05/29 18:12:52.930 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:53.022 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:53.036 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:53.039 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:53.103 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/29 18:12:53.104 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/29 18:12:53.157 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/29 18:12:53.158 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] Starting CDC state checker... [2024/05/29 18:12:53.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:53.247 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/29 18:12:53.248 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/29 18:12:53.261 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/29 18:12:53.341 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4e3508eb_60a5_4725_897d_04847a563611"] [2024/05/29 18:12:53.370 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:53.459 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:53.472 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/29 18:12:53.541 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9b84412c_a36e_444f_a162_ecc12c378c22"] [2024/05/29 18:12:53.548 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0139bcb0_1f94_42c7_8602_c8b59b5072cf"] [2024/05/29 18:12:53.504 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/29 18:12:53.561 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/29 18:12:53.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/29 18:12:53.747 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:53.830 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:53.929 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:53.958 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:54.031 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/29 18:12:54.057 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:54.061 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:54.124 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:54.127 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later [2024/05/29 18:12:54.344 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:54.363 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:54.464 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:54.537 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later [2024/05/29 18:12:54.631 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:54.638 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:54.643 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:54.644 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49e40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25660, start at 2024-05-29 18:12:50.449752646 +0800 CST m=+4.391547840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:12:54.826 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:54.842 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:54.943 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:54.975 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:55.068 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:55.121 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:55.127 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:55.132 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:55.152 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa9574947_8ccd_4874_a428_979303ca31d0"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:55.336 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:55.342 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:55.456 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:55.545 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:55.649 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:55.649 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:55.653 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:55.662 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:55.748 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/29 18:12:55.751 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/29 18:12:55.854 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:55.862 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:56.034 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:56.135 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:56.243 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:56.245 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:56.247 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:56.328 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:56.355 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/29 18:12:56.355 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/29 18:12:56.457 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:56.533 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Starting CDC state checker... table capture_suicide_while_balance_table.t4 exists [2024/05/29 18:12:56.629 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:56.664 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:56.830 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:56.834 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:56.839 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:56.859 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:56.924 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/29 18:12:56.928 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/29 18:12:56.960 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:56.967 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:57.046 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa1180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25992, start at 2024-05-29 18:12:56.036046106 +0800 CST m=+5.260603338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:56.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:56.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:56.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log [2024/05/29 18:12:57.135 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:57.326 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:57.329 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:57.332 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:57.361 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:12:57.432 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/29 18:12:57.448 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/29 18:12:57.522 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:57.525 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:57.545 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:12:57.634 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:57.831 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:57.834 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:57.838 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:57.923 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:57.937 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/29 18:12:57.963 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/29 18:12:58.024 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:58.033 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:58.039 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e938001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25122, start at 2024-05-29 18:12:27.900108616 +0800 CST m=+4.380669690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:12:58.331 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:58.339 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:58.352 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:58.452 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:58.462 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/29 18:12:58.538 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/29 18:12:58.558 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:58.561 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:58.753 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:58.755 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:12:58.765 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:58.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/29 18:12:58.941 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/29 18:12:59.036 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/29 18:12:59.060 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:59.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] Starting CDC state checker... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 139 100 35 100 104 792 2355 --:--:-- --:--:-- --:--:-- 2363 { "status": true, "message": "" Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:59.339 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:59.342 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:12:59.349 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:12:59.389 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:59.428 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/29 18:12:59.466 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/29 18:12:59.626 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:59.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:12:59.639 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:59.677 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/29 18:12:59.714 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/29 18:12:59.763 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/29 18:12:59.892 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/29 18:12:59.899 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:12:59.980 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/29 18:13:00.019 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:12:59.574 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/29 18:12:59.781 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/29 18:12:59.947 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/29 18:13:00.077 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/29 18:13:00.078 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/29 18:13:00.231 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/29 18:13:00.233 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:13:00.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/29 18:13:00.274 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/29 18:13:00.297 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/29 18:13:00.433 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:13:00.502 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/29 18:13:00.549 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/29 18:13:00.550 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/29 18:13:00.742 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/29 18:13:00.744 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/29 18:13:00.682 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m11.532922673s"] [2024/05/29 18:13:00.780 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m11.630832656s"] table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later [2024/05/29 18:13:00.810 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/29 18:13:00.979 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] }lease 22318fc3d582d4b1 revoked check diff successfully cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later [2024/05/29 18:13:01.122 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/29 18:13:01.123 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596f0c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25437, start at 2024-05-29 18:12:36.427976226 +0800 CST m=+4.813037569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/29 18:13:01.513 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/29 18:13:01.516 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/29 18:13:01.542 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:13:01 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:13:02.018 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/29 18:13:02.020 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/29 18:13:02.035 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 450095359984926736 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Wed May 29 18:12:50 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Wed May 29 18:12:50 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml check diff failed 1-th time, retry later check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5943900017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24777, start at 2024-05-29 18:12:33.669357471 +0800 CST m=+4.395852463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:33.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:33.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:33.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:13:02.334 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/29 18:13:02.520 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/29 18:13:02.523 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/29 18:13:02.630 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/29 18:13:02.631 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/29 18:13:02.792 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/29 18:13:02.832 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] check diff successfully table mark.finish_mark_2 not exists for 2-th check, retry later [2024/05/29 18:13:03.250 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/29 18:13:03.253 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/29 18:13:03.255 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/29 18:13:03.256 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] check diff successfully [Wed May 29 18:13:03 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:13:03 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> [2024/05/29 18:13:03.726 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/29 18:13:03.744 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:13:04.150 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/29 18:13:04.152 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/29 18:13:04.232 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/29 18:13:04.236 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] table mark.finish_mark_2 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b20180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25983, start at 2024-05-29 18:13:04.157942274 +0800 CST m=+4.294666303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log table mark.finish_mark_2 not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... table mark.finish_mark_2 not exists for 5-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_2 not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:13:11.059 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.76452557s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_2 not exists for 7-th check, retry later table mark.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 1-th time, retry later table mark.finish_mark_2 not exists for 8-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49e40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25660, start at 2024-05-29 18:12:50.449752646 +0800 CST m=+4.391547840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... skip some SQLs in tidb v4.0.x table common_1.v1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_2 not exists for 9-th check, retry later table common_1.v1 not exists for 2-th check, retry later Verifying downstream PD is started... table mark.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:13:18 CST 2024] <<<<<< run test case multi_source success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a38340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26154, start at 2024-05-29 18:12:49.329124295 +0800 CST m=+4.397083952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table mark.finish_mark_2 not exists for 10-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa1180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25992, start at 2024-05-29 18:12:56.036046106 +0800 CST m=+5.260603338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:56.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:56.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:56.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.recover_and_insert not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596e34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25344, start at 2024-05-29 18:12:36.38439708 +0800 CST m=+4.828833547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.365 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.365 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f596f0c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:25437, start at 2024-05-29 18:12:36.427976226 +0800 CST m=+4.813037569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:36.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:36.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:36.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:13:21 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.2938029382.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f594b1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24676, start at 2024-05-29 18:12:34.129465982 +0800 CST m=+4.906168365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:34.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:34.119 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:34.119 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5943900017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:24777, start at 2024-05-29 18:12:33.669357471 +0800 CST m=+4.395852463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:33.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:33.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:33.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:13:22 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.2908129083.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 2-th check, retry later Starting Upstream TiDB... table common_1.recover_and_insert not exists for 3-th check, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b269594b-1b33-467f-82b0-114f51a9ad24 {"id":"b269594b-1b33-467f-82b0-114f51a9ad24","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d58e b269594b-1b33-467f-82b0-114f51a9ad24 /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b269594b-1b33-467f-82b0-114f51a9ad24 {"id":"b269594b-1b33-467f-82b0-114f51a9ad24","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d58e b269594b-1b33-467f-82b0-114f51a9ad24 /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b269594b-1b33-467f-82b0-114f51a9ad24 {"id":"b269594b-1b33-467f-82b0-114f51a9ad24","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d58e b269594b-1b33-467f-82b0-114f51a9ad24 /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 3-th check, retry later table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:25 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 29086 owner id 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e default [Wed May 29 18:13:26 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.2922429226.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e7f8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25029, start at 2024-05-29 18:12:27.814508035 +0800 CST m=+4.386256664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f58e938001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:25122, start at 2024-05-29 18:12:27.900108616 +0800 CST m=+4.380669690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:27.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:27.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:27.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:13:26 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2921329215.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.finish_mark not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:29 GMT < Content-Length: 911 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e {"id":"3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06332 3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "3e4b7f01-a7bd-4219-90d1-c5cdfc1b898e default" | grep id "id": "6896ca73-8a8a-4782-9e51-08f0c0200395", run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7eaca46-973f-4761-ba76-44d9e2065c86 {"id":"c7eaca46-973f-4761-ba76-44d9e2065c86","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea85b5 c7eaca46-973f-4761-ba76-44d9e2065c86 /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7eaca46-973f-4761-ba76-44d9e2065c86 {"id":"c7eaca46-973f-4761-ba76-44d9e2065c86","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea85b5 c7eaca46-973f-4761-ba76-44d9e2065c86 /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7eaca46-973f-4761-ba76-44d9e2065c86 {"id":"c7eaca46-973f-4761-ba76-44d9e2065c86","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea85b5 c7eaca46-973f-4761-ba76-44d9e2065c86 /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.29279.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager capture_id: 6896ca73-8a8a-4782-9e51-08f0c0200395 pass test: check health Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7374362379040519986,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-29T18:13:30.20732887+08:00","start_ts":450095378616287234,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095378616287234,"checkpoint_ts":450095378616287234,"checkpoint_time":"2024-05-29 18:13:30.078"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... pass test: get status coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: create changefeed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5caa8c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21396, start at 2024-05-29 18:13:29.396153621 +0800 CST m=+4.296706840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.379 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.379 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log table common_1.finish_mark exists table mark.finish_mark_3 not exists for 6-th check, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb8540019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23666, start at 2024-05-29 18:13:30.308807184 +0800 CST m=+4.379502294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.316 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:13:32 CST 2024] <<<<<< run test case common_1 success! >>>>>> table autorandom_test.table_a not exists for 1-th check, retry later table autorandom_test.table_a exists check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:13:26 CST 2024] <<<<<< run test case autorandom success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 table mark.finish_mark_3 not exists for 7-th check, retry later You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b20180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25983, start at 2024-05-29 18:13:04.157942274 +0800 CST m=+4.294666303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce66c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31244, start at 2024-05-29 18:13:33.214006839 +0800 CST m=+4.289780651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D3EE0B3CB58383 < X-Xss-Protection: 1; mode=block < Date: Wed, 29 May 2024 10:13:34 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49180004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25573, start at 2024-05-29 18:12:50.379316626 +0800 CST m=+4.396932652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a49e40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:25660, start at 2024-05-29 18:12:50.449752646 +0800 CST m=+4.391547840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:50.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:50.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:50.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.29725.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Starting CDC state checker... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450095380624834561 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095380624834561 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa0280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25921, start at 2024-05-29 18:12:55.949499343 +0800 CST m=+5.278241323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:55.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:55.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:55.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5aa1180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:25992, start at 2024-05-29 18:12:56.036046106 +0800 CST m=+5.260603338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:56.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:56.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:56.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30092.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095381297758209 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095381297758209 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 11-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a3a9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26062, start at 2024-05-29 18:12:49.4696893 +0800 CST m=+4.629877612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... table mark.finish_mark_3 not exists for 12-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5a38340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:26154, start at 2024-05-29 18:12:49.329124295 +0800 CST m=+4.397083952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:14:49.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:12:49.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:02:49.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:13:43 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.3011130113.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:46 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.30222.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 9accb250-bea9-4aa2-b22e-7bd05bba535d Info: {"upstream_id":7374362476699326044,"namespace":"default","id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:13:47.080558029+08:00","start_ts":450095383055958018,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095383055958018,"checkpoint_ts":450095383055958018,"checkpoint_time":"2024-05-29 18:13:47.014"} PASS table mark.finish_mark_3 not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dca180009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25280, start at 2024-05-29 18:13:47.7941702 +0800 CST m=+4.553906961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.801 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.782 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.782 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 15-th check, retry later table big_txn.finish_mark_1 exists check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Wed May 29 18:13:39 CST 2024] <<<<<< run test case big_txn success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:13:48 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd6ac000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33567, start at 2024-05-29 18:13:48.603286725 +0800 CST m=+4.253846369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 16-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:13:52 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b1e840019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25890, start at 2024-05-29 18:13:04.070865765 +0800 CST m=+4.297443123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5b20180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--tf44s, pid:25983, start at 2024-05-29 18:13:04.157942274 +0800 CST m=+4.294666303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:04.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:04.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:13:53 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.3011030112.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 17-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5caa8c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21396, start at 2024-05-29 18:13:29.396153621 +0800 CST m=+4.296706840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.379 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.379 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... table mark.finish_mark_3 not exists for 18-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:13:55 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:13:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e6758a4-abf5-40ea-88ba-18bcbcfbb480 {"id":"3e6758a4-abf5-40ea-88ba-18bcbcfbb480","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d763098e 3e6758a4-abf5-40ea-88ba-18bcbcfbb480 /tidb/cdc/default/default/upstream/7374362538375475847 {"id":7374362538375475847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e6758a4-abf5-40ea-88ba-18bcbcfbb480 {"id":"3e6758a4-abf5-40ea-88ba-18bcbcfbb480","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d763098e 3e6758a4-abf5-40ea-88ba-18bcbcfbb480 /tidb/cdc/default/default/upstream/7374362538375475847 {"id":7374362538375475847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e6758a4-abf5-40ea-88ba-18bcbcfbb480 {"id":"3e6758a4-abf5-40ea-88ba-18bcbcfbb480","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d763098e 3e6758a4-abf5-40ea-88ba-18bcbcfbb480 /tidb/cdc/default/default/upstream/7374362538375475847 {"id":7374362538375475847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30540, start at 2024-05-29 18:13:54.94092126 +0800 CST m=+4.341857547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.918 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.918 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 19-th check, retry later Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce66c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31244, start at 2024-05-29 18:13:33.214006839 +0800 CST m=+4.289780651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb8540019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23666, start at 2024-05-29 18:13:30.308807184 +0800 CST m=+4.379502294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.316 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:13:59 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> table mark.finish_mark_3 not exists for 20-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 21-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 22-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 23-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... table mark.finish_mark_3 not exists for 24-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 25-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 26-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ca9e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21303, start at 2024-05-29 18:13:29.345371751 +0800 CST m=+4.319871752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5caa8c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:21396, start at 2024-05-29 18:13:29.396153621 +0800 CST m=+4.296706840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:29.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:29.379 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:29.379 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25363.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095389649403905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095389649403905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095389649403905 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:14:12 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2540225404.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 27-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying downstream PD is started... table mark.finish_mark_3 not exists for 28-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:15 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c4aabf0-80ce-4962-8639-3940935ecf22 {"id":"6c4aabf0-80ce-4962-8639-3940935ecf22","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7aec68e 6c4aabf0-80ce-4962-8639-3940935ecf22 /tidb/cdc/default/default/upstream/7374362643517619053 {"id":7374362643517619053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c4aabf0-80ce-4962-8639-3940935ecf22 {"id":"6c4aabf0-80ce-4962-8639-3940935ecf22","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7aec68e 6c4aabf0-80ce-4962-8639-3940935ecf22 /tidb/cdc/default/default/upstream/7374362643517619053 {"id":7374362643517619053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c4aabf0-80ce-4962-8639-3940935ecf22 {"id":"6c4aabf0-80ce-4962-8639-3940935ecf22","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7aec68e 6c4aabf0-80ce-4962-8639-3940935ecf22 /tidb/cdc/default/default/upstream/7374362643517619053 {"id":7374362643517619053,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095389649403905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25472.out cli changefeed create --start-ts=450095389649403905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374362643517619053,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:14:15.907162929+08:00","start_ts":450095389649403905,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095389649403905,"checkpoint_ts":450095389649403905,"checkpoint_time":"2024-05-29 18:14:12.166"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 12974 0 --:--:-- --:--:-- --:--:-- 13812 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:14:12.166","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-29 18:14:16.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:14:12.166","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:14:16.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:14:12.166","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:14:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-29 18:14:12.166' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:14:12.166","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:14:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:14:12.166","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:14:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-29 18:14:16' + echo 'sink_checkpoint_ts is 2024-05-29' 18:14:12.166 sink_checkpoint_ts is 2024-05-29 18:14:12.166 ++ date -d '2024-05-29 18:14:12.166' +%s + checkpoint_timestamp=1716977652 ++ date -d '2024-05-29 18:14:16' +%s + current_timestamp=1716977656 + '[' 4 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dca180009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25280, start at 2024-05-29 18:13:47.7941702 +0800 CST m=+4.553906961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.801 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.782 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.782 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 29-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb7880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23487, start at 2024-05-29 18:13:30.242768896 +0800 CST m=+6.289664923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5cb8540019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:23666, start at 2024-05-29 18:13:30.308807184 +0800 CST m=+4.379502294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:30.316 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:30.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:30.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:14:17 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.2760627608.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f92ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25438, start at 2024-05-29 18:14:17.044784575 +0800 CST m=+5.539123338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30540, start at 2024-05-29 18:13:54.94092126 +0800 CST m=+4.341857547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.918 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.918 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ***************** properties ***************** "mysql.db"="consistent_replicate_storage_file_large_value" "readallfields"="true" "scanproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "fieldlengthdistribution"="constant" "recordcount"="100" "readproportion"="0" "mysql.user"="root" "threadcount"="10" "fieldcount"="1000" "workload"="core" "mysql.host"="127.0.0.1" "dotransactions"="false" "fieldlength"="15000" "updateproportion"="0" "insertproportion"="0" "mysql.port"="4000" ********************************************** + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9db2124e-0dab-4f04-9789-1b64908ec2f5 {"id":"9db2124e-0dab-4f04-9789-1b64908ec2f5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7c19f8e 9db2124e-0dab-4f04-9789-1b64908ec2f5 /tidb/cdc/default/default/upstream/7374362641616282088 {"id":7374362641616282088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9db2124e-0dab-4f04-9789-1b64908ec2f5 {"id":"9db2124e-0dab-4f04-9789-1b64908ec2f5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7c19f8e 9db2124e-0dab-4f04-9789-1b64908ec2f5 /tidb/cdc/default/default/upstream/7374362641616282088 {"id":7374362641616282088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9db2124e-0dab-4f04-9789-1b64908ec2f5 {"id":"9db2124e-0dab-4f04-9789-1b64908ec2f5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7c19f8e 9db2124e-0dab-4f04-9789-1b64908ec2f5 /tidb/cdc/default/default/upstream/7374362641616282088 {"id":7374362641616282088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd6ac000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33567, start at 2024-05-29 18:13:48.603286725 +0800 CST m=+4.253846369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 30-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce5880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31142, start at 2024-05-29 18:13:33.182290868 +0800 CST m=+4.339126753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5ce66c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:31244, start at 2024-05-29 18:13:33.214006839 +0800 CST m=+4.289780651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:33.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:33.211 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:33.211 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.35346.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095392091013121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095392091013121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:14:21 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.3538635388.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 31-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1195058-24b3-409a-8147-35b3b0a0e927 {"id":"c1195058-24b3-409a-8147-35b3b0a0e927","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7cb5790 c1195058-24b3-409a-8147-35b3b0a0e927 /tidb/cdc/default/default/upstream/7374362663525519839 {"id":7374362663525519839,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1195058-24b3-409a-8147-35b3b0a0e927 {"id":"c1195058-24b3-409a-8147-35b3b0a0e927","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7cb5790 c1195058-24b3-409a-8147-35b3b0a0e927 /tidb/cdc/default/default/upstream/7374362663525519839 {"id":7374362663525519839,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1195058-24b3-409a-8147-35b3b0a0e927 {"id":"c1195058-24b3-409a-8147-35b3b0a0e927","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7cb5790 c1195058-24b3-409a-8147-35b3b0a0e927 /tidb/cdc/default/default/upstream/7374362663525519839 {"id":7374362663525519839,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.35448.out cli changefeed create --start-ts=450095392091013121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table mark.finish_mark_3 not exists for 32-th check, retry later Create changefeed successfully! ID: 95448b5a-cbc8-4004-afe3-984dbf77abd9 Info: {"upstream_id":7374362663525519839,"namespace":"default","id":"95448b5a-cbc8-4004-afe3-984dbf77abd9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:14:25.221288084+08:00","start_ts":450095392091013121,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095392091013121,"checkpoint_ts":450095392091013121,"checkpoint_time":"2024-05-29 18:14:21.480"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.simple1 not exists for 1-th check, retry later table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (29086) - No such process wait process 29086 exit for 1-th time... process 29086 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 6896ca73-8a8a-4782-9e51-08f0c0200395 -A1 | grep '"is-owner": true' "is-owner": true, run task successfully [Wed May 29 18:14:27 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.2955829560.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 33-th check, retry later table mark.finish_mark_3 not exists for 34-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} Status:0xc00118eae0 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} Status:0xc00118eb40 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0864ab32-4d2b-440f-b1b2-c39dc9478417 {"id":"0864ab32-4d2b-440f-b1b2-c39dc9478417","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f0640a 0864ab32-4d2b-440f-b1b2-c39dc9478417 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.766068968+08:00","start-ts":450095378786418693,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095393243660294} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450095391592153130,"min-table-barrier-ts":450095391736332322,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} Status:0xc00118eae0 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} Status:0xc00118eb40 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0864ab32-4d2b-440f-b1b2-c39dc9478417 {"id":"0864ab32-4d2b-440f-b1b2-c39dc9478417","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f0640a 0864ab32-4d2b-440f-b1b2-c39dc9478417 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.766068968+08:00","start-ts":450095378786418693,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095393243660294} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450095391592153130,"min-table-barrier-ts":450095391736332322,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} Status:0xc00118eae0 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} Status:0xc00118eb40 TaskPositions:map[0864ab32-4d2b-440f-b1b2-c39dc9478417:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 6896ca73-8a8a-4782-9e51-08f0c0200395:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0864ab32-4d2b-440f-b1b2-c39dc9478417 {"id":"0864ab32-4d2b-440f-b1b2-c39dc9478417","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/6896ca73-8a8a-4782-9e51-08f0c0200395 {"id":"6896ca73-8a8a-4782-9e51-08f0c0200395","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f06350 6896ca73-8a8a-4782-9e51-08f0c0200395 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6f0640a 0864ab32-4d2b-440f-b1b2-c39dc9478417 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.647005895+08:00","start-ts":450095378747097093,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378760204293} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.766068968+08:00","start-ts":450095378786418693,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095393243660294} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7374362408898739516,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-29T18:13:30.850760051+08:00","start-ts":450095378812633099,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378812633103} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450095391592153130,"min-table-barrier-ts":450095391736332322,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450095394292236291,"min-table-barrier-ts":450095394292236295,"admin-job-type":0} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/0864ab32-4d2b-440f-b1b2-c39dc9478417/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6896ca73-8a8a-4782-9e51-08f0c0200395/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362408898739516 {"id":7374362408898739516,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' "address": "127.0.0.1:8300", run task successfully pass test: list changefeed INSERT_ERROR - Takes(s): 8.3, Count: 53, OPS: 6.4, Avg(us): 1120532, Min(us): 379722, Max(us): 2451535, 95th(us): 0, 99th(us): 0 start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... pass test: get changefeed table mark.finish_mark_3 not exists for 35-th check, retry later pass test: resume changefeed pass test: pause changefeed start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... pass test: rebalance table pass test: move table start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... pass test: get processors pass test: list processors pass test: set log level table mark.finish_mark_3 not exists for 36-th check, retry later pass test: remove changefeed pass test: resign owner Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 37-th check, retry later Run finished, takes 15.906844742s INSERT_ERROR - Takes(s): 14.2, Count: 100, OPS: 7.0, Avg(us): 953277, Min(us): 251964, Max(us): 2451535, 95th(us): 0, 99th(us): 0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table mark.finish_mark_3 not exists for 38-th check, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 29218 exit for 1-th time... wait process 29218 exit for 2-th time... wait process 29218 exit for 3-th time... wait process 29218 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (29218) - No such process wait process 29218 exit for 5-th time... process 29218 already exit [Wed May 29 18:14:36 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2940429406.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 39-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30455, start at 2024-05-29 18:13:54.931862623 +0800 CST m=+4.390663347 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.902 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.902 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5e39980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:30540, start at 2024-05-29 18:13:54.94092126 +0800 CST m=+4.341857547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:54.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:54.918 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:54.918 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:14:39 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3447134473.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd5f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33492, start at 2024-05-29 18:13:48.579782968 +0800 CST m=+4.287024525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dd6ac000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:33567, start at 2024-05-29 18:13:48.603286725 +0800 CST m=+4.253846369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:48.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:48.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:48.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.37349.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095396957454337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095396957454337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc000dff008 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392509657100,"min-table-barrier-ts":450095392509657100,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc000dff008 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392509657100,"min-table-barrier-ts":450095392509657100,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc000dff008 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392509657100,"min-table-barrier-ts":450095392509657100,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Wed May 29 18:14:40 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2946829470.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 40-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dc8900018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25203, start at 2024-05-29 18:13:47.709825123 +0800 CST m=+4.526761159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.684 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.684 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5dca180009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:25280, start at 2024-05-29 18:13:47.7941702 +0800 CST m=+4.553906961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:15:47.801 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:13:47.782 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:03:47.782 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:14:42 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.2924829250.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 {"id":"9ddaaefa-9d3c-415f-b7dd-e1caa32f1142","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815aa90 9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 {"id":"9ddaaefa-9d3c-415f-b7dd-e1caa32f1142","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815aa90 9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 {"id":"9ddaaefa-9d3c-415f-b7dd-e1caa32f1142","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815aa90 9ddaaefa-9d3c-415f-b7dd-e1caa32f1142 /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 41-th check, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later table owner_resign.t1 exists check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc00082f098 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392640991258,"min-table-barrier-ts":450095392640991258,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc00082f098 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/d+ grep -q 'failed to get info:' efault/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392640991258,"min-table-barrier-ts":450095392640991258,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} Status:0xc00082f098 TaskPositions:map[f3a81766-5329-4cc3-bac6-b1dd380aae9a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3a81766-5329-4cc3-bac6-b1dd380aae9a {"id":"f3a81766-5329-4cc3-bac6-b1dd380aae9a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6ea881f f3a81766-5329-4cc3-bac6-b1dd380aae9a /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374362379040519986,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:13:30.20732887+08:00","start-ts":450095378616287234,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095378642501637} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450095392640991258,"min-table-barrier-ts":450095392640991258,"admin-job-type":0} /tidb/cdc/default/default/task/position/f3a81766-5329-4cc3-bac6-b1dd380aae9a/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362379040519986 {"id":7374362379040519986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2075cca8-b085-4d09-bb77-09f561d47d96 {"id":"2075cca8-b085-4d09-bb77-09f561d47d96","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81cad88 2075cca8-b085-4d09-bb77-09f561d47d96 /tidb/cdc/default/default/upstream/7374362727458124473 {"id":7374362727458124473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2075cca8-b085-4d09-bb77-09f561d47d96 {"id":"2075cca8-b085-4d09-bb77-09f561d47d96","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81cad88 2075cca8-b085-4d09-bb77-09f561d47d96 /tidb/cdc/default/default/upstream/7374362727458124473 {"id":7374362727458124473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2075cca8-b085-4d09-bb77-09f561d47d96 {"id":"2075cca8-b085-4d09-bb77-09f561d47d96","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81cad88 2075cca8-b085-4d09-bb77-09f561d47d96 /tidb/cdc/default/default/upstream/7374362727458124473 {"id":7374362727458124473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7374362727458124473,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:14:45.430642291+08:00","start_ts":450095397510316033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095397510316033,"checkpoint_ts":450095397510316033,"checkpoint_time":"2024-05-29 18:14:42.153"} ***************** properties ***************** "updateproportion"="0" "recordcount"="10000" "dotransactions"="false" "operationcount"="0" "workload"="core" "threadcount"="10" "mysql.user"="root" "scanproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "insertproportion"="0" "readallfields"="true" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.db"="move_table" ********************************************** Run finished, takes 1.793764416s INSERT - Takes(s): 1.8, Count: 10000, OPS: 5591.9, Avg(us): 1747, Min(us): 761, Max(us): 37526, 95th(us): 2000, 99th(us): 35000 [Wed May 29 18:14:37 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2987729879.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:40 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.29937.out cli changefeed create --start-ts=450095380624834561 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info: {"upstream_id":7374362477924431331,"namespace":"default","id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:14:40.912627424+08:00","start_ts":450095380624834561,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095380624834561,"checkpoint_ts":450095380624834561,"checkpoint_time":"2024-05-29 18:13:37.740"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Wed May 29 18:14:41 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2998229984.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc002b23560 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395790127116,"min-table-barrier-ts":450095397677563913,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc002b23560 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395790127116,"min-table-barrier-ts":450095397677563913,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc002b23560 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395790127116,"min-table-barrier-ts":450095397677563913,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Wed May 29 18:14:44 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.3005930061.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7374362727458124473,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:14:45.628465514+08:00","start_ts":450095397510316033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095397510316033,"checkpoint_ts":450095397510316033,"checkpoint_time":"2024-05-29 18:14:42.153"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7374362727458124473,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:14:45.778946099+08:00","start_ts":450095397510316033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095397510316033,"checkpoint_ts":450095397510316033,"checkpoint_time":"2024-05-29 18:14:42.153"} table ddl_manager.finish_mark not exists for 2-th check, retry later check diff successfully [Wed May 29 18:14:46 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.3045030452.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later table mark.finish_mark_3 not exists for 43-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc000b305a0 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 94e819c1-8124-459c-b796-60bb516e787d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/94e819c1-8124-459c-b796-60bb516e787d {"id":"94e819c1-8124-459c-b796-60bb516e787d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262364 94e819c1-8124-459c-b796-60bb516e787d /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395934044227,"min-table-barrier-ts":450095398733479943,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/94e819c1-8124-459c-b796-60bb516e787d/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc000b305a0 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 94e819c1-8124-459c-b796-60bb516e787d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/94e819c1-8124-459c-b796-60bb516e787d {"id":"94e819c1-8124-459c-b796-60bb516e787d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262364 94e819c1-8124-459c-b796-60bb516e787d /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395934044227,"min-table-barrier-ts":450095398733479943,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/94e819c1-8124-459c-b796-60bb516e787d/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {ClusterID:default ID:default/16717ec5-8b0a-4db5-964c-95b1fee0ed6a Info:{"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} Status:0xc000b305a0 TaskPositions:map[6b942cfe-01d0-47eb-ae0f-fead571a3caa:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 94e819c1-8124-459c-b796-60bb516e787d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d63be2d5-6158-4fb3-8050-839cad044259:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b942cfe-01d0-47eb-ae0f-fead571a3caa {"id":"6b942cfe-01d0-47eb-ae0f-fead571a3caa","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/94e819c1-8124-459c-b796-60bb516e787d {"id":"94e819c1-8124-459c-b796-60bb516e787d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d63be2d5-6158-4fb3-8050-839cad044259 {"id":"d63be2d5-6158-4fb3-8050-839cad044259","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262317 d63be2d5-6158-4fb3-8050-839cad044259 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d726234d 6b942cfe-01d0-47eb-ae0f-fead571a3caa /tidb/cdc/default/__cdc_meta__/owner/22318fc3d7262364 94e819c1-8124-459c-b796-60bb516e787d /tidb/cdc/default/default/changefeed/info/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"upstream-id":7374362477924431331,"namespace":"default","changefeed-id":"16717ec5-8b0a-4db5-964c-95b1fee0ed6a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:14:40.912627424+08:00","start-ts":450095380624834561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397179228164} /tidb/cdc/default/default/changefeed/status/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":450095395934044227,"min-table-barrier-ts":450095398733479943,"admin-job-type":0} /tidb/cdc/default/default/task/position/6b942cfe-01d0-47eb-ae0f-fead571a3caa/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/94e819c1-8124-459c-b796-60bb516e787d/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d63be2d5-6158-4fb3-8050-839cad044259/16717ec5-8b0a-4db5-964c-95b1fee0ed6a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362477924431331 {"id":7374362477924431331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 3-th check, retry later table move_table.usertable exists [Wed May 29 18:14:38 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.3020430206.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/899d4b18-8ead-448d-ab70-e129f0abc0e8 {"id":"899d4b18-8ead-448d-ab70-e129f0abc0e8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d72d5e19 899d4b18-8ead-448d-ab70-e129f0abc0e8 /tidb/cdc/default/default/upstream/7374362500916538321 {"id":7374362500916538321,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/899d4b18-8ead-448d-ab70-e129f0abc0e8 {"id":"899d4b18-8ead-448d-ab70-e129f0abc0e8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d72d5e19 899d4b18-8ead-448d-ab70-e129f0abc0e8 /tidb/cdc/default/default/upstream/7374362500916538321 {"id":7374362500916538321,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/899d4b18-8ead-448d-ab70-e129f0abc0e8 {"id":"899d4b18-8ead-448d-ab70-e129f0abc0e8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d72d5e19 899d4b18-8ead-448d-ab70-e129f0abc0e8 /tidb/cdc/default/default/upstream/7374362500916538321 {"id":7374362500916538321,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30259.out cli changefeed create --start-ts=450095381297758209 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 155cbb2d-9d25-4d59-8b5a-9af3048a72b3 Info: {"upstream_id":7374362500916538321,"namespace":"default","id":"155cbb2d-9d25-4d59-8b5a-9af3048a72b3","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:14:42.50685302+08:00","start_ts":450095381297758209,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095381297758209,"checkpoint_ts":450095381297758209,"checkpoint_time":"2024-05-29 18:13:40.307"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9accb250-bea9-4aa2-b22e-7bd05bba535d {ClusterID:default ID:default/9accb250-bea9-4aa2-b22e-7bd05bba535d Info:{"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} Status:0xc0009b7ce0 TaskPositions:map[14fcad68-0d53-4801-b932-85fa7a94a17d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7f23cb2-f533-40fc-813e-64f264406629:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fcad68-0d53-4801-b932-85fa7a94a17d {"id":"14fcad68-0d53-4801-b932-85fa7a94a17d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c5dd 14fcad68-0d53-4801-b932-85fa7a94a17d /tidb/cdc/default/default/changefeed/info/9accb250-bea9-4aa2-b22e-7bd05bba535d {"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} /tidb/cdc/default/default/changefeed/status/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":450095399203766280,"min-table-barrier-ts":450095399203766280,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fcad68-0d53-4801-b932-85fa7a94a17d/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7f23cb2-f533-40fc-813e-64f264406629/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/9accb250-bea9-4aa2-b22e-7bd05bba535d {ClusterID:default ID:default/9accb250-bea9-4aa2-b22e-7bd05bba535d Info:{"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} Status:0xc0009b7ce0 TaskPositions:map[14fcad68-0d53-4801-b932-85fa7a94a17d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7f23cb2-f533-40fc-813e-64f264406629:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fcad68-0d53-4801-b932-85fa7a94a17d {"id":"14fcad68-0d53-4801-b932-85fa7a94a17d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c5dd 14fcad68-0d53-4801-b932-85fa7a94a17d /tidb/cdc/default/default/changefeed/info/9accb250-bea9-4aa2-b22e-7bd05bba535d {"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} /tidb/cdc/default/default/changefeed/status/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":450095399203766280,"min-table-barrier-ts":450095399203766280,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fcad68-0d53-4801-b932-85fa7a94a17d/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7f23cb2-f533-40fc-813e-64f264406629/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/9accb250-bea9-4aa2-b22e-7bd05bba535d {ClusterID:default ID:default/9accb250-bea9-4aa2-b22e-7bd05bba535d Info:{"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} Status:0xc0009b7ce0 TaskPositions:map[14fcad68-0d53-4801-b932-85fa7a94a17d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7f23cb2-f533-40fc-813e-64f264406629:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14fcad68-0d53-4801-b932-85fa7a94a17d {"id":"14fcad68-0d53-4801-b932-85fa7a94a17d","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7f23cb2-f533-40fc-813e-64f264406629 {"id":"b7f23cb2-f533-40fc-813e-64f264406629","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c490 b7f23cb2-f533-40fc-813e-64f264406629 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d739c5dd 14fcad68-0d53-4801-b932-85fa7a94a17d /tidb/cdc/default/default/changefeed/info/9accb250-bea9-4aa2-b22e-7bd05bba535d {"upstream-id":7374362476699326044,"namespace":"default","changefeed-id":"9accb250-bea9-4aa2-b22e-7bd05bba535d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:47.080558029+08:00","start-ts":450095383055958018,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095383068803075} /tidb/cdc/default/default/changefeed/status/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":450095399203766280,"min-table-barrier-ts":450095399203766280,"admin-job-type":0} /tidb/cdc/default/default/task/position/14fcad68-0d53-4801-b932-85fa7a94a17d/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7f23cb2-f533-40fc-813e-64f264406629/9accb250-bea9-4aa2-b22e-7bd05bba535d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362476699326044 {"id":7374362476699326044,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table mark.finish_mark_3 not exists for 44-th check, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:14:50 CST 2024] <<<<<< run test case simple success! >>>>>> table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 45-th check, retry later check_changefeed_state http://127.0.0.1:2379 d9e99382-449c-4b2b-a45f-2d97683befaa finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=d9e99382-449c-4b2b-a45f-2d97683befaa + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d9e99382-449c-4b2b-a45f-2d97683befaa -s + info='{ "upstream_id": 7374362098489403720, "namespace": "default", "id": "d9e99382-449c-4b2b-a45f-2d97683befaa", "state": "finished", "checkpoint_tso": 450095383282974723, "checkpoint_time": "2024-05-29 18:13:47.880", "error": null }' + echo '{ "upstream_id": 7374362098489403720, "namespace": "default", "id": "d9e99382-449c-4b2b-a45f-2d97683befaa", "state": "finished", "checkpoint_tso": 450095383282974723, "checkpoint_time": "2024-05-29 18:13:47.880", "error": null }' { "upstream_id": 7374362098489403720, "namespace": "default", "id": "d9e99382-449c-4b2b-a45f-2d97683befaa", "state": "finished", "checkpoint_tso": 450095383282974723, "checkpoint_time": "2024-05-29 18:13:47.880", "error": null } ++ echo '{' '"upstream_id":' 7374362098489403720, '"namespace":' '"default",' '"id":' '"d9e99382-449c-4b2b-a45f-2d97683befaa",' '"state":' '"finished",' '"checkpoint_tso":' 450095383282974723, '"checkpoint_time":' '"2024-05-29' '18:13:47.880",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7374362098489403720, '"namespace":' '"default",' '"id":' '"d9e99382-449c-4b2b-a45f-2d97683befaa",' '"state":' '"finished",' '"checkpoint_tso":' 450095383282974723, '"checkpoint_time":' '"2024-05-29' '18:13:47.880",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:14:46 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table ddl_manager.finish_mark not exists for 5-th check, retry later table consistent_replicate_storage_file_large_value.t2 exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 6-th check, retry later table mark.finish_mark_3 not exists for 46-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:14:53 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.2974529747.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 47-th check, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:14:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/0307fee7-a99b-43f8-9f49-599b71fa09d6 {ClusterID:default ID:default/0307fee7-a99b-43f8-9f49-599b71fa09d6 Info:{"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} Status:0xc000a53458 TaskPositions:map[3399a86b-3584-424a-90a4-dc48d8c73c6b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 94, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 91, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 92, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 87, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 88, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 82, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 84, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3399a86b-3584-424a-90a4-dc48d8c73c6b {"id":"3399a86b-3584-424a-90a4-dc48d8c73c6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d763 3399a86b-3584-424a-90a4-dc48d8c73c6b /tidb/cdc/default/default/changefeed/info/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} /tidb/cdc/default/default/changefeed/status/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":450095401025667078,"min-table-barrier-ts":450095401287548935,"admin-job-type":0} /tidb/cdc/default/default/task/position/3399a86b-3584-424a-90a4-dc48d8c73c6b/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0307fee7-a99b-43f8-9f49-599b71fa09d6 {ClusterID:default ID:default/0307fee7-a99b-43f8-9f49-599b71fa09d6 Info:{"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} Status:0xc000a53458 TaskPositions:map[3399a86b-3584-424a-90a4-dc48d8c73c6b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 94, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 91, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 92, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 87, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 88, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 82, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 84, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3399a86b-3584-424a-90a4-dc48d8c73c6b {"id":"3399a86b-3584-424a-90a4-dc48d8c73c6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d763 3399a86b-3584-424a-90a4-dc48d8c73c6b /tidb/cdc/default/default/changefeed/info/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} /tidb/cdc/default/default/changefeed/status/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":450095401025667078,"min-table-barrier-ts":450095401287548935,"admin-job-type":0} /tidb/cdc/default/default/task/position/3399a86b-3584-424a-90a4-dc48d8c73c6b/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0307fee7-a99b-43f8-9f49-599b71fa09d6 {ClusterID:default ID:default/0307fee7-a99b-43f8-9f49-599b71fa09d6 Info:{"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} Status:0xc000a53458 TaskPositions:map[3399a86b-3584-424a-90a4-dc48d8c73c6b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 94, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 91, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 92, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 87, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 88, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 82, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating tableID: 84, resolvedTs: 450095401025667079, checkpointTs: 450095401025667078, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3399a86b-3584-424a-90a4-dc48d8c73c6b {"id":"3399a86b-3584-424a-90a4-dc48d8c73c6b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d6e2d763 3399a86b-3584-424a-90a4-dc48d8c73c6b /tidb/cdc/default/default/changefeed/info/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"upstream-id":7374362420438734855,"namespace":"default","changefeed-id":"0307fee7-a99b-43f8-9f49-599b71fa09d6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:13:24.590012138+08:00","start-ts":450095377144348677,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095377170563076} /tidb/cdc/default/default/changefeed/status/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":450095401025667078,"min-table-barrier-ts":450095401287548935,"admin-job-type":0} /tidb/cdc/default/default/task/position/3399a86b-3584-424a-90a4-dc48d8c73c6b/0307fee7-a99b-43f8-9f49-599b71fa09d6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362420438734855 {"id":7374362420438734855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 48-th check, retry later [2024/05/29 18:14:56.199 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/29 18:14:56.201 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/29 18:14:56.201 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=d63be2d5-6158-4fb3-8050-839cad044259] [2024/05/29 18:14:56.202 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/29 18:14:56.202 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/29 18:14:56.316 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=6b942cfe-01d0-47eb-ae0f-fead571a3caa] [processorDetail="{\"table_ids\":[]}"] [2024/05/29 18:14:56.466 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=94e819c1-8124-459c-b796-60bb516e787d] [processorDetail="{\"table_ids\":[]}"] [2024/05/29 18:14:56.615 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=d63be2d5-6158-4fb3-8050-839cad044259] [processorDetail="{\"table_ids\":[82,84]}"] [2024/05/29 18:14:56.615 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"6b942cfe-01d0-47eb-ae0f-fead571a3caa\":[],\"94e819c1-8124-459c-b796-60bb516e787d\":[],\"d63be2d5-6158-4fb3-8050-839cad044259\":[{\"ID\":82,\"Changefeed\":\"16717ec5-8b0a-4db5-964c-95b1fee0ed6a\"},{\"ID\":84,\"Changefeed\":\"16717ec5-8b0a-4db5-964c-95b1fee0ed6a\"}]}"] [2024/05/29 18:14:56.615 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=16717ec5-8b0a-4db5-964c-95b1fee0ed6a&target-cp-id=94e819c1-8124-459c-b796-60bb516e787d&table-id=82"] [2024/05/29 18:14:56.666 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=82] [2024/05/29 18:14:56.666 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=16717ec5-8b0a-4db5-964c-95b1fee0ed6a&target-cp-id=94e819c1-8124-459c-b796-60bb516e787d&table-id=84"] [2024/05/29 18:14:56.715 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=84] [2024/05/29 18:14:56.715 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=d63be2d5-6158-4fb3-8050-839cad044259] [targetCapture=94e819c1-8124-459c-b796-60bb516e787d] table move_table.check1 exists check diff successfully table ddl_manager.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f92ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25438, start at 2024-05-29 18:14:17.044784575 +0800 CST m=+5.539123338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 9-th check, retry later table mark.finish_mark_3 not exists for 49-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 10-th check, retry later table mark.finish_mark_3 not exists for 50-th check, retry later [2024/05/29 18:15:01.477 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/29 18:15:01.478 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/29 18:15:01.479 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=d63be2d5-6158-4fb3-8050-839cad044259] [2024/05/29 18:15:01.480 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/29 18:15:01.480 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/29 18:15:01.615 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=6b942cfe-01d0-47eb-ae0f-fead571a3caa] [processorDetail="{\"table_ids\":[86]}"] [2024/05/29 18:15:01.766 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=94e819c1-8124-459c-b796-60bb516e787d] [processorDetail="{\"table_ids\":[84]}"] [2024/05/29 18:15:01.915 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=16717ec5-8b0a-4db5-964c-95b1fee0ed6a] [captureID=d63be2d5-6158-4fb3-8050-839cad044259] [processorDetail="{\"table_ids\":[]}"] [2024/05/29 18:15:01.916 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"6b942cfe-01d0-47eb-ae0f-fead571a3caa\":[{\"ID\":86,\"Changefeed\":\"16717ec5-8b0a-4db5-964c-95b1fee0ed6a\"}],\"94e819c1-8124-459c-b796-60bb516e787d\":[{\"ID\":84,\"Changefeed\":\"16717ec5-8b0a-4db5-964c-95b1fee0ed6a\"}],\"d63be2d5-6158-4fb3-8050-839cad044259\":[]}"] [2024/05/29 18:15:01.916 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=16717ec5-8b0a-4db5-964c-95b1fee0ed6a&target-cp-id=d63be2d5-6158-4fb3-8050-839cad044259&table-id=86"] [2024/05/29 18:15:01.966 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=86] [2024/05/29 18:15:01.966 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=6b942cfe-01d0-47eb-ae0f-fead571a3caa] [targetCapture=d63be2d5-6158-4fb3-8050-839cad044259] check diff successfully table move_table.check2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later table mark.finish_mark_3 not exists for 51-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table move_table.check2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:04 CST 2024] <<<<<< run test case move_table success! >>>>>> table mark.finish_mark_3 not exists for 52-th check, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later Verifying downstream PD is started... pass test: get tso cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:04 CST 2024] <<<<<< run test case http_api success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:05 CST 2024] <<<<<< run test case owner_resign success! >>>>>> table mark.finish_mark_3 not exists for 53-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 54-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 14-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 55-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 56-th check, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:15:14 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> table mark.finish_mark_3 not exists for 57-th check, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630f580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31412, start at 2024-05-29 18:15:14.178714842 +0800 CST m=+4.695643796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_stop_delay.t not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 58-th check, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 10973 0 --:--:-- --:--:-- --:--:-- 11045 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:15:16.066","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-29 18:15:10.016","now_ts":"2024-05-29 18:15:17.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:15:16.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:15:10.016","now_ts":"2024-05-29' '18:15:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:15:16.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:15:10.016","now_ts":"2024-05-29' '18:15:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f63314c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:31057, start at 2024-05-29 18:15:16.313963236 +0800 CST m=+4.552965153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.320 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log table processor_stop_delay.t exists # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_redo_resolved_ts 0307fee7-a99b-43f8-9f49-599b71fa09d6 450095406962966530 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/0307fee7-a99b-43f8-9f49-599b71fa09d6/meta global resolved ts 450095406281392133 not forward to 450095406962966530 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f94380010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25353, start at 2024-05-29 18:14:17.123194216 +0800 CST m=+5.720368252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f5f92ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:25438, start at 2024-05-29 18:14:17.044784575 +0800 CST m=+5.539123338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:16:17.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:14:17.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:04:17.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:15:19 CST 2024] <<<<<< START cdc server in canal_json_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.2944429446.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' check diff successfully + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later Starting CDC state checker... table ddl_manager.finish_mark not exists for 19-th check, retry later check_redo_resolved_ts 0307fee7-a99b-43f8-9f49-599b71fa09d6 450095406962966530 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/0307fee7-a99b-43f8-9f49-599b71fa09d6/meta check diff failed 1-th time, retry later global resolved ts 450095406805942277 not forward to 450095406962966530 run task failed 2-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark exists table ddl_manager.finish_mark not exists for 20-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:15:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/468ef494-346b-4670-8980-1602dbb249de {"id":"468ef494-346b-4670-8980-1602dbb249de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8ae828e 468ef494-346b-4670-8980-1602dbb249de /tidb/cdc/default/default/upstream/7374362839351302448 {"id":7374362839351302448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/468ef494-346b-4670-8980-1602dbb249de {"id":"468ef494-346b-4670-8980-1602dbb249de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8ae828e 468ef494-346b-4670-8980-1602dbb249de /tidb/cdc/default/default/upstream/7374362839351302448 {"id":7374362839351302448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/468ef494-346b-4670-8980-1602dbb249de {"id":"468ef494-346b-4670-8980-1602dbb249de","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8ae828e 468ef494-346b-4670-8980-1602dbb249de /tidb/cdc/default/default/upstream/7374362839351302448 {"id":7374362839351302448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.cli.29513.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: 48662ad0-e37a-42c5-b8f0-f5694c60fcc4 Info: {"upstream_id":7374362839351302448,"namespace":"default","id":"48662ad0-e37a-42c5-b8f0-f5694c60fcc4","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-29T18:15:22.430674764+08:00","start_ts":450095408045096965,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095408045096965,"checkpoint_ts":450095408045096965,"checkpoint_time":"2024-05-29 18:15:22.340"} PASS start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:22 CST 2024] <<<<<< run test case default_value success! >>>>>> table ddl_manager.finish_mark not exists for 21-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check_redo_resolved_ts 0307fee7-a99b-43f8-9f49-599b71fa09d6 450095406962966530 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/0307fee7-a99b-43f8-9f49-599b71fa09d6/meta run task successfully check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/29 18:15:25.121 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:15:25.121 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:15:25.122 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095407854518277] [resolvedTs=450095408378806276] [2024/05/29 18:15:25.122 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=290.564ยตs] [2024/05/29 18:15:25.122 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=292.734ยตs] [2024/05/29 18:15:25.122 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:15:25.122 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:15:25.127 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:15:25.129 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:15:25.130 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:15:25.130 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:15:25.130 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:15:25.133 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:15:25.135 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:15:25.135 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=450095408378806276] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:15:25.135 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] Apply redo log successfully check diff successfully [Wed May 29 18:15:25 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 table ddl_manager.finish_mark not exists for 22-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:31 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> table ddl_manager.finish_mark not exists for 25-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:28 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 26-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32343, start at 2024-05-29 18:15:32.194477743 +0800 CST m=+4.364209801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428980005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30686, start at 2024-05-29 18:15:32.138349904 +0800 CST m=+4.270948196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6444d40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31597, start at 2024-05-29 18:15:33.943286062 +0800 CST m=+4.571776970 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.941 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.941 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log table ddl_manager.finish_mark not exists for 27-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D3EE279183036A < X-Xss-Protection: 1; mode=block < Date: Wed, 29 May 2024 10:15:36 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Starting CDC state checker... Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting CDC state checker... table ddl_manager.finish_mark not exists for 28-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:15:37 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630f580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31412, start at 2024-05-29 18:15:14.178714842 +0800 CST m=+4.695643796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table2.t2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table2.t2 not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 30-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table2.t2 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table2.t2 not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 32-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table2.t2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 33-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f63314c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:31057, start at 2024-05-29 18:15:16.313963236 +0800 CST m=+4.552965153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.320 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... run test case test_kill_owner [Wed May 29 18:15:39 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3746937471.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:15:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.37515.out cli changefeed create --start-ts=450095396957454337 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 33146874-74af-4ab6-a7c9-794a2ee8c8dc Info: {"upstream_id":7374362733485705949,"namespace":"default","id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:15:42.826594229+08:00","start_ts":450095396957454337,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095396957454337,"checkpoint_ts":450095396957454337,"checkpoint_time":"2024-05-29 18:14:40.044"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 37474 owner id 3155c066-d002-44e2-b997-f16733a57cb1 [Wed May 29 18:15:43 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3762137623.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:15:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0007f7db8 TaskPositions:map[3155c066-d002-44e2-b997-f16733a57cb1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae928356-5574-4eae-8432-eb32db1f146d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/ae928356-5574-4eae-8432-eb32db1f146d {"id":"ae928356-5574-4eae-8432-eb32db1f146d","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819855e ae928356-5574-4eae-8432-eb32db1f146d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412253294607,"min-table-barrier-ts":450095412384628768,"admin-job-type":0} /tidb/cdc/default/default/task/position/3155c066-d002-44e2-b997-f16733a57cb1/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae928356-5574-4eae-8432-eb32db1f146d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0007f7db8 TaskPositions:map[3155c066-d002-44e2-b997-f16733a57cb1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae928356-5574-4eae-8432-eb32db1f146d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/ae928356-5574-4eae-8432-eb32db1f146d {"id":"ae928356-5574-4eae-8432-eb32db1f146d","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819855e ae928356-5574-4eae-8432-eb32db1f146d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412253294607,"min-table-barrier-ts":450095412384628768,"admin-job-type":0} /tidb/cdc/default/default/task/position/3155c066-d002-44e2-b997-f16733a57cb1/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae928356-5574-4eae-8432-eb32db1f146d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0007f7db8 TaskPositions:map[3155c066-d002-44e2-b997-f16733a57cb1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae928356-5574-4eae-8432-eb32db1f146d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3155c066-d002-44e2-b997-f16733a57cb1 {"id":"3155c066-d002-44e2-b997-f16733a57cb1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/ae928356-5574-4eae-8432-eb32db1f146d {"id":"ae928356-5574-4eae-8432-eb32db1f146d","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198528 3155c066-d002-44e2-b997-f16733a57cb1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819855e ae928356-5574-4eae-8432-eb32db1f146d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412253294607,"min-table-barrier-ts":450095412384628768,"admin-job-type":0} /tidb/cdc/default/default/task/position/3155c066-d002-44e2-b997-f16733a57cb1/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae928356-5574-4eae-8432-eb32db1f146d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "3155c066-d002-44e2-b997-f16733a57cb1" | grep -v cluster_id | grep id "id": "ae928356-5574-4eae-8432-eb32db1f146d", "cluster-id": "default" "cluster-id": "default" run task successfully capture_id: ae928356-5574-4eae-8432-eb32db1f146d /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (37474) - No such process wait process 37474 exit for 1-th time... process 37474 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep ae928356-5574-4eae-8432-eb32db1f146d -A1 | grep '"is-owner": true' "is-owner": true, run task successfully test_kill_owner: pass start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table2.t2 not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 34-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_owner [Wed May 29 18:15:50 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3779837800.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table2.t2 not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6534e0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24265, start at 2024-05-29 18:15:49.315386073 +0800 CST m=+4.263348660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.321 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31169, start at 2024-05-29 18:15:50.996211144 +0800 CST m=+4.578045512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:51.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.972 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.972 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table2.t2 not exists for 9-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:15:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000cd7458 TaskPositions:map[b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 82, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 84, resolvedTs: 450095412384628768, checkpointTs: 450095412384628768, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412384628768,"min-table-barrier-ts":450095416146395139,"admin-job-type":0} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000cd7458 TaskPositions:map[b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 82, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 84, resolvedTs: 450095412384628768, checkpointTs: 450095412384628768, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412384628768,"min-table-barrier-ts":450095416146395139,"admin-job-type":0} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000cd7458 TaskPositions:map[b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 82, resolvedTs: 450095416146395139, checkpointTs: 450095415884251139, state: Replicating tableID: 84, resolvedTs: 450095412384628768, checkpointTs: 450095412384628768, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095412384628768,"min-table-barrier-ts":450095416146395139,"admin-job-type":0} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 37803 owner id b7567550-a3f3-4b07-adfb-17ab5a80d240 [Wed May 29 18:15:53 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3794337945.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 36-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table partition_table2.t2 exists table ddl_manager.finish_mark not exists for 37-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428980005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30686, start at 2024-05-29 18:15:32.138349904 +0800 CST m=+4.270948196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:15:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0019c36c8 TaskPositions:map[888895c9-7b61-4ac6-ab74-9fa30233645a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/888895c9-7b61-4ac6-ab74-9fa30233645a {"id":"888895c9-7b61-4ac6-ab74-9fa30233645a","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985b0 888895c9-7b61-4ac6-ab74-9fa30233645a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095416932564996,"min-table-barrier-ts":450095416932564996,"admin-job-type":0} /tidb/cdc/default/default/task/position/888895c9-7b61-4ac6-ab74-9fa30233645a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0019c36c8 TaskPositions:map[888895c9-7b61-4ac6-ab74-9fa30233645a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/888895c9-7b61-4ac6-ab74-9fa30233645a {"id":"888895c9-7b61-4ac6-ab74-9fa30233645a","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985b0 888895c9-7b61-4ac6-ab74-9fa30233645a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} + grep -q 'failed to get info:' /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095416932564996,"min-table-barrier-ts":450095416932564996,"admin-job-type":0} /tidb/cdc/default/default/task/position/888895c9-7b61-4ac6-ab74-9fa30233645a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0019c36c8 TaskPositions:map[888895c9-7b61-4ac6-ab74-9fa30233645a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b7567550-a3f3-4b07-adfb-17ab5a80d240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/888895c9-7b61-4ac6-ab74-9fa30233645a {"id":"888895c9-7b61-4ac6-ab74-9fa30233645a","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/b7567550-a3f3-4b07-adfb-17ab5a80d240 {"id":"b7567550-a3f3-4b07-adfb-17ab5a80d240","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819858f b7567550-a3f3-4b07-adfb-17ab5a80d240 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985b0 888895c9-7b61-4ac6-ab74-9fa30233645a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095416932564996,"min-table-barrier-ts":450095416932564996,"admin-job-type":0} /tidb/cdc/default/default/task/position/888895c9-7b61-4ac6-ab74-9fa30233645a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b7567550-a3f3-4b07-adfb-17ab5a80d240/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "b7567550-a3f3-4b07-adfb-17ab5a80d240" | grep id "cluster-id": "default" "id": "888895c9-7b61-4ac6-ab74-9fa30233645a", "cluster-id": "default" run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630cec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31319, start at 2024-05-29 18:15:14.01825496 +0800 CST m=+4.643250636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f630f580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:31412, start at 2024-05-29 18:15:14.178714842 +0800 CST m=+4.695643796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:14.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:14.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:14.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:15:57 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.3540035402.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 capture_id: 888895c9-7b61-4ac6-ab74-9fa30233645a ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '888895c9-7b61-4ac6-ab74-9fa30233645a' 888895c9-7b61-4ac6-ab74-9fa30233645a run task successfully test_hang_up_owner: pass table ddl_manager.finish_mark not exists for 38-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_owner [Wed May 29 18:15:58 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3810838110.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32343, start at 2024-05-29 18:15:32.194477743 +0800 CST m=+4.364209801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 39-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:16:00 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3471434716.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e808379-d7cd-47f4-877d-de7b96c71a04 {"id":"3e808379-d7cd-47f4-877d-de7b96c71a04","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d947ea8e 3e808379-d7cd-47f4-877d-de7b96c71a04 /tidb/cdc/default/default/upstream/7374363091806897099 {"id":7374363091806897099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e808379-d7cd-47f4-877d-de7b96c71a04 {"id":"3e808379-d7cd-47f4-877d-de7b96c71a04","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d947ea8e 3e808379-d7cd-47f4-877d-de7b96c71a04 /tidb/cdc/default/default/upstream/7374363091806897099 {"id":7374363091806897099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e808379-d7cd-47f4-877d-de7b96c71a04 {"id":"3e808379-d7cd-47f4-877d-de7b96c71a04","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d947ea8e 3e808379-d7cd-47f4-877d-de7b96c71a04 /tidb/cdc/default/default/upstream/7374363091806897099 {"id":7374363091806897099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0014a1230 TaskPositions:map[f49e0321-a271-4984-86f5-c52e4abd0986:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 82, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 84, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f49e0321-a271-4984-86f5-c52e4abd0986 {"id":"f49e0321-a271-4984-86f5-c52e4abd0986","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985dd f49e0321-a271-4984-86f5-c52e4abd0986 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095417194971140,"min-table-barrier-ts":450095417981403140,"admin-job-type":0} /tidb/cdc/default/default/task/position/f49e0321-a271-4984-86f5-c52e4abd0986/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0014a1230 TaskPositions:map[f49e0321-a271-4984-86f5-c52e4abd0986:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 82, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 84, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f49e0321-a271-4984-86f5-c52e4abd0986 {"id":"f49e0321-a271-4984-86f5-c52e4abd0986","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985dd f49e0321-a271-4984-86f5-c52e4abd0986 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095417194971140,"min-table-barrier-ts":450095417981403140,"admin-job-type":0} /tidb/cdc/default/default/task/position/f49e0321-a271-4984-86f5-c52e4abd0986/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0014a1230 TaskPositions:map[f49e0321-a271-4984-86f5-c52e4abd0986:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 82, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing tableID: 84, resolvedTs: 450095417194971140, checkpointTs: 450095417194971140, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f49e0321-a271-4984-86f5-c52e4abd0986 {"id":"f49e0321-a271-4984-86f5-c52e4abd0986","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985dd f49e0321-a271-4984-86f5-c52e4abd0986 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095417194971140,"min-table-barrier-ts":450095417981403140,"admin-job-type":0} /tidb/cdc/default/default/task/position/f49e0321-a271-4984-86f5-c52e4abd0986/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 + grep -q 'etcd info' {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65db100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29259, start at 2024-05-29 18:15:59.964148883 +0800 CST m=+4.365912722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:59.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:59.940 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:59.940 +0800 All versions after safe point can be accessed. (DO NOT EDIT) owner pid: 38113 owner id f49e0321-a271-4984-86f5-c52e4abd0986 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'f49e0321-a271-4984-86f5-c52e4abd0986' /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81985dd run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log [2024/05/29 18:15:50.324 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 33"] "is-owner": true, run task successfully test_expire_owner pass table ddl_manager.finish_mark not exists for 40-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Wed May 29 18:16:02 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3832038322.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... table ddl_manager.finish_mark not exists for 41-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {ClusterID:default ID:default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 Info:{"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} Status:0xc0015c1ba8 TaskPositions:map[c68efbd9-ae78-48e7-83df-cbd53e0765ef:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 95, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 97, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 84, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 83, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 101, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 91, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 94, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 87, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 85, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 90, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 92, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68efbd9-ae78-48e7-83df-cbd53e0765ef {"id":"c68efbd9-ae78-48e7-83df-cbd53e0765ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815ac34 c68efbd9-ae78-48e7-83df-cbd53e0765ef /tidb/cdc/default/default/changefeed/info/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} /tidb/cdc/default/default/changefeed/status/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":450095418561527811,"min-table-barrier-ts":450095418823671811,"admin-job-type":0} /tidb/cdc/default/default/task/position/c68efbd9-ae78-48e7-83df-cbd53e0765ef/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {ClusterID:default ID:default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 Info:{"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} Status:0xc0015c1ba8 TaskPositions:map[c68efbd9-ae78-48e7-83df-cbd53e0765ef:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 95, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 97, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 84, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 83, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 101, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 91, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 94, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 87, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 85, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 90, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 92, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68efbd9-ae78-48e7-83df-cbd53e0765ef {"id":"c68efbd9-ae78-48e7-83df-cbd53e0765ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815ac34 c68efbd9-ae78-48e7-83df-cbd53e0765ef /tidb/cdc/default/default/changefeed/info/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} /tidb/cdc/default/default/changefeed/status/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":450095418561527811,"min-table-barrier-ts":450095418823671811,"admin-job-type":0} /tidb/cdc/default/default/task/position/c68efbd9-ae78-48e7-83df-cbd53e0765ef/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {ClusterID:default ID:default/a3b960a2-6f22-4254-ad8e-62b60ab02b58 Info:{"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} Status:0xc0015c1ba8 TaskPositions:map[c68efbd9-ae78-48e7-83df-cbd53e0765ef:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 95, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 97, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 84, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 83, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 101, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 91, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 94, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 87, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 85, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 90, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating tableID: 92, resolvedTs: 450095418561527812, checkpointTs: 450095418561527811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68efbd9-ae78-48e7-83df-cbd53e0765ef {"id":"c68efbd9-ae78-48e7-83df-cbd53e0765ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d815ac34 c68efbd9-ae78-48e7-83df-cbd53e0765ef /tidb/cdc/default/default/changefeed/info/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"upstream-id":7374362749267825590,"namespace":"default","changefeed-id":"a3b960a2-6f22-4254-ad8e-62b60ab02b58","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:14:42.643618281+08:00","start-ts":450095397615960068,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095397629067269} /tidb/cdc/default/default/changefeed/status/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":450095418561527811,"min-table-barrier-ts":450095418823671811,"admin-job-type":0} /tidb/cdc/default/default/task/position/c68efbd9-ae78-48e7-83df-cbd53e0765ef/a3b960a2-6f22-4254-ad8e-62b60ab02b58 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362749267825590 {"id":7374362749267825590,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000b594a0 TaskPositions:map[cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 80, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 82, resolvedTs: 450095419292123140, checkpointTs: 450095419292123140, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095419292123140,"min-table-barrier-ts":450095419292123140,"admin-job-type":0} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000b594a0 TaskPositions:map[cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 80, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 82, resolvedTs: 450095419292123140, checkpointTs: 450095419292123140, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095419292123140,"min-table-barrier-ts":450095419292123140,"admin-job-type":0} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000b594a0 TaskPositions:map[cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 80, resolvedTs: 450095419318075396, checkpointTs: 450095419292123140, state: Replicating tableID: 82, resolvedTs: 450095419292123140, checkpointTs: 450095419292123140, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-per+ grep -q 'etcd info' centage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095419292123140,"min-table-barrier-ts":450095419292123140,"admin-job-type":0} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6444d40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31597, start at 2024-05-29 18:15:33.943286062 +0800 CST m=+4.571776970 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.941 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.941 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... "is-owner": true, run task successfully Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 38325 owner id cbf32873-e962-4fd6-9918-924bfc7b517a [Wed May 29 18:16:06 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3846538467.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 not exists for 1-th check, retry later table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095417210437634, "checkpoint_time": "2024-05-29 18:15:57.303", "error": null }' + echo '{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095417210437634, "checkpoint_time": "2024-05-29 18:15:57.303", "error": null }' { "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095417210437634, "checkpoint_time": "2024-05-29 18:15:57.303", "error": null } ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450095417210437634, '"checkpoint_time":' '"2024-05-29' '18:15:57.303",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450095417210437634, '"checkpoint_time":' '"2024-05-29' '18:15:57.303",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095417354878988, "checkpoint_time": "2024-05-29 18:15:57.854", "error": null }' + echo '{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095417354878988, "checkpoint_time": "2024-05-29 18:15:57.854", "error": null }' { "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095417354878988, "checkpoint_time": "2024-05-29 18:15:57.854", "error": null } ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450095417354878988, '"checkpoint_time":' '"2024-05-29' '18:15:57.854",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450095417354878988, '"checkpoint_time":' '"2024-05-29' '18:15:57.854",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095415899979777, "checkpoint_time": "2024-05-29 18:15:52.304", "error": { "time": "2024-05-29T18:15:53.409458803+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095416017944582]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095415899979777, "checkpoint_time": "2024-05-29 18:15:52.304", "error": { "time": "2024-05-29T18:15:53.409458803+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095416017944582]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7374362727458124473, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095415899979777, "checkpoint_time": "2024-05-29 18:15:52.304", "error": { "time": "2024-05-29T18:15:53.409458803+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095416017944582]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450095415899979777, '"checkpoint_time":' '"2024-05-29' '18:15:52.304",' '"error":' '{' '"time":' '"2024-05-29T18:15:53.409458803+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450095416017944582]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374362727458124473, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450095415899979777, '"checkpoint_time":' '"2024-05-29' '18:15:52.304",' '"error":' '{' '"time":' '"2024-05-29T18:15:53.409458803+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450095416017944582]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095416017944582]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:15:52.754 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095416017944582. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095416017944582]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:15:59 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT table ddl_manager.finish_mark not exists for 43-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:16:08 CST 2024] <<<<<< run test case multi_topics success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0037a7110 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420078555143,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0037a7110 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420078555143,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0037a7110 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use+ grep -q 'etcd info' -file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420078555143,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "cbf32873-e962-4fd6-9918-924bfc7b517a" | grep id "cluster-id": "default" "id": "192a5dd4-7353-4d44-a9ee-6a6556676f0e", "cluster-id": "default" run task successfully capture_id: 192a5dd4-7353-4d44-a9ee-6a6556676f0e start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 44-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f632ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:30986, start at 2024-05-29 18:15:16.164693174 +0800 CST m=+4.477674280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f63314c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:31057, start at 2024-05-29 18:15:16.313963236 +0800 CST m=+4.552965153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:16.320 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:16.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:16.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:16:10 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.3505135053.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 45-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_with_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... 0 [Wed May 29 18:16:13 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3863138633.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:13 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b8ac69f-b225-4e38-8503-d9b432b6fd75 {"id":"6b8ac69f-b225-4e38-8503-d9b432b6fd75","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d96c0191 6b8ac69f-b225-4e38-8503-d9b432b6fd75 /tidb/cdc/default/default/upstream/7374363105272850673 {"id":7374363105272850673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b8ac69f-b225-4e38-8503-d9b432b6fd75 {"id":"6b8ac69f-b225-4e38-8503-d9b432b6fd75","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d96c0191 6b8ac69f-b225-4e38-8503-d9b432b6fd75 /tidb/cdc/default/default/upstream/7374363105272850673 {"id":7374363105272850673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b8ac69f-b225-4e38-8503-d9b432b6fd75 {"id":"6b8ac69f-b225-4e38-8503-d9b432b6fd75","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d96c0191 6b8ac69f-b225-4e38-8503-d9b432b6fd75 /tidb/cdc/default/default/upstream/7374363105272850673 {"id":7374363105272850673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 46-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31169, start at 2024-05-29 18:15:50.996211144 +0800 CST m=+4.578045512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:51.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.972 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.972 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 47-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000666fa8 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb8c5eef-d77d-404d-928e-0f3470f6e6a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cb8c5eef-d77d-404d-928e-0f3470f6e6a3 {"id":"cb8c5eef-d77d-404d-928e-0f3470f6e6a3","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819865d cb8c5eef-d77d-404d-928e-0f3470f6e6a3 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420340437000,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb8c5eef-d77d-404d-928e-0f3470f6e6a3/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000666fa8 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb8c5eef-d77d-404d-928e-0f3470f6e6a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cb8c5eef-d77d-404d-928e-0f3470f6e6a3 {"id":"cb8c5eef-d77d-404d-928e-0f3470f6e6a3","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819865d cb8c5eef-d77d-404d-928e-0f3470f6e6a3 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420340437000,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb8c5eef-d77d-404d-928e-0f3470f6e6a3/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000666fa8 TaskPositions:map[192a5dd4-7353-4d44-a9ee-6a6556676f0e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb8c5eef-d77d-404d-928e-0f3470f6e6a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cbf32873-e962-4fd6-9918-924bfc7b517a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/192a5dd4-7353-4d44-a9ee-6a6556676f0e {"id":"192a5dd4-7353-4d44-a9ee-6a6556676f0e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cb8c5eef-d77d-404d-928e-0f3470f6e6a3 {"id":"cb8c5eef-d77d-404d-928e-0f3470f6e6a3","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/cbf32873-e962-4fd6-9918-924bfc7b517a {"id":"cbf32873-e962-4fd6-9918-924bfc7b517a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198602 cbf32873-e962-4fd6-9918-924bfc7b517a /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198629 192a5dd4-7353-4d44-a9ee-6a6556676f0e /tidb/cdc/default/__cdc_meta__/owner/22318fc3d819865d cb8c5eef-d77d-404d-928e-0f3470f6e6a3 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095420340437000,"min-table-barrier-ts":450095420340437000,"admin-job-type":0} /tidb/cdc/default/default/task/position/192a5dd4-7353-4d44-a9ee-6a6556676f0e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb8c5eef-d77d-404d-928e-0f3470f6e6a3/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbf32873-e962-4fd6-9918-924bfc7b517a/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 48-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f64273c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32243, start at 2024-05-29 18:15:32.074415061 +0800 CST m=+4.338018532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:32343, start at 2024-05-29 18:15:32.194477743 +0800 CST m=+4.364209801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.36335.out cli tso query --pd=http://127.0.0.1:2379 cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' + set +x + tso='450095422674567169 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095422674567169 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + workdir=/tmp/tidb_cdc_test/canal_json_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_basic ++ date + echo '[Wed May 29 18:16:17 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>>' [Wed May 29 18:16:17 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 49-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6534e0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24265, start at 2024-05-29 18:15:49.315386073 +0800 CST m=+4.263348660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.321 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 50-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6427e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30599, start at 2024-05-29 18:15:32.096518538 +0800 CST m=+4.277787452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6428980005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:30686, start at 2024-05-29 18:15:32.138349904 +0800 CST m=+4.270948196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:32.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:32.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:32.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 table ddl_manager.finish_mark not exists for 51-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 52-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6443500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31536, start at 2024-05-29 18:15:33.877279926 +0800 CST m=+4.589379283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6444d40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:31597, start at 2024-05-29 18:15:33.943286062 +0800 CST m=+4.571776970 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:33.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:33.941 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:33.941 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.35591.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later + set +x + tso='450095425051426817 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095425051426817 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:16:27 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.3563835640.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65db100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29259, start at 2024-05-29 18:15:59.964148883 +0800 CST m=+4.365912722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:59.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:59.940 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:59.940 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 53-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6780a40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30793, start at 2024-05-29 18:16:26.956996802 +0800 CST m=+4.397207003 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/error.log Starting TLS TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_retryable_error [Wed May 29 18:16:29 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3889438896.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 54-th check, retry later Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d96a64-ac0f-4bc8-85b5-b127833e6f62 {"id":"53d96a64-ac0f-4bc8-85b5-b127833e6f62","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9b45590 53d96a64-ac0f-4bc8-85b5-b127833e6f62 /tidb/cdc/default/default/upstream/7374363181863423938 {"id":7374363181863423938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d96a64-ac0f-4bc8-85b5-b127833e6f62 {"id":"53d96a64-ac0f-4bc8-85b5-b127833e6f62","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9b45590 53d96a64-ac0f-4bc8-85b5-b127833e6f62 /tidb/cdc/default/default/upstream/7374363181863423938 {"id":7374363181863423938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d96a64-ac0f-4bc8-85b5-b127833e6f62 {"id":"53d96a64-ac0f-4bc8-85b5-b127833e6f62","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9b45590 53d96a64-ac0f-4bc8-85b5-b127833e6f62 /tidb/cdc/default/default/upstream/7374363181863423938 {"id":7374363181863423938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.35697.out cli changefeed create --start-ts=450095425051426817 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 5d4008e6-758e-486a-b2f3-d0f91e0bd442 Info: {"upstream_id":7374363181863423938,"namespace":"default","id":"5d4008e6-758e-486a-b2f3-d0f91e0bd442","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:16:30.966470766+08:00","start_ts":450095425051426817,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095425051426817,"checkpoint_ts":450095425051426817,"checkpoint_time":"2024-05-29 18:16:27.214"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 55-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00048eff0 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 82, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 84, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095425059291142,"min-table-barrier-ts":450095426369748999,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00048eff0 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 82, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 84, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095425059291142,"min-table-barrier-ts":450095426369748999,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00048eff0 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 82, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing tableID: 84, resolvedTs: 450095425059291142, checkpointTs: 450095425059291142, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095425059291142,"min-table-barrier-ts":450095426369748999,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 38899 owner id 7f0a0294-0299-4f07-b1e2-fe64729d6112 [Wed May 29 18:16:33 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3904439046.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31069, start at 2024-05-29 18:15:50.900031287 +0800 CST m=+4.539664598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:50.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f654ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:31169, start at 2024-05-29 18:15:50.996211144 +0800 CST m=+4.578045512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:51.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:50.972 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:50.972 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:16:34 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3523335235.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67e714000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:35017, start at 2024-05-29 18:16:33.493622921 +0800 CST m=+4.370026770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:33.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:33.477 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:33.477 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 57-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 2 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 check diff failed 6-th time, retry later Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003e2d128 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d1c13ad9-543c-401f-bb60-59116bd5e802:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d1c13ad9-543c-401f-bb60-59116bd5e802 {"id":"d1c13ad9-543c-401f-bb60-59116bd5e802","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986cb d1c13ad9-543c-401f-bb60-59116bd5e802 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427169288193,"min-table-barrier-ts":450095427195502594,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c13ad9-543c-401f-bb60-59116bd5e802/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003e2d128 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d1c13ad9-543c-401f-bb60-59116bd5e802:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d1c13ad9-543c-401f-bb60-59116bd5e802 {"id":"d1c13ad9-543c-401f-bb60-59116bd5e802","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986cb d1c13ad9-543c-401f-bb60-59116bd5e802 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427169288193,"min-table-barrier-ts":450095427195502594,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c13ad9-543c-401f-bb60-59116bd5e802/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003e2d128 TaskPositions:map[7f0a0294-0299-4f07-b1e2-fe64729d6112:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d1c13ad9-543c-401f-bb60-59116bd5e802:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f0a0294-0299-4f07-b1e2-fe64729d6112 {"id":"7f0a0294-0299-4f07-b1e2-fe64729d6112","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/d1c13ad9-543c-401f-bb60-59116bd5e802 {"id":"d1c13ad9-543c-401f-bb60-59116bd5e802","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986aa 7f0a0294-0299-4f07-b1e2-fe64729d6112 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81986cb d1c13ad9-543c-401f-bb60-59116bd5e802 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} + grep -q 'etcd info' /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427169288193,"min-table-barrier-ts":450095427195502594,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f0a0294-0299-4f07-b1e2-fe64729d6112/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1c13ad9-543c-401f-bb60-59116bd5e802/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "7f0a0294-0299-4f07-b1e2-fe64729d6112" | grep id "cluster-id": "default" "id": "d1c13ad9-543c-401f-bb60-59116bd5e802", "cluster-id": "default" run task successfully capture_id: d1c13ad9-543c-401f-bb60-59116bd5e802 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 38759 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '7f0a0294-0299-4f07-b1e2-fe64729d6112' 7f0a0294-0299-4f07-b1e2-fe64729d6112 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Wed May 29 18:16:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3922439226.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 58-th check, retry later check diff successfully [Wed May 29 18:16:38 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f365ba0a-16a7-4553-9c56-70805525acf0 {"id":"f365ba0a-16a7-4553-9c56-70805525acf0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72b88 f365ba0a-16a7-4553-9c56-70805525acf0 /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f365ba0a-16a7-4553-9c56-70805525acf0 {"id":"f365ba0a-16a7-4553-9c56-70805525acf0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72b88 f365ba0a-16a7-4553-9c56-70805525acf0 /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f365ba0a-16a7-4553-9c56-70805525acf0 {"id":"f365ba0a-16a7-4553-9c56-70805525acf0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72b88 f365ba0a-16a7-4553-9c56-70805525acf0 /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting TLS TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67e714000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:35017, start at 2024-05-29 18:16:33.493622921 +0800 CST m=+4.370026770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:33.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:33.477 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:33.477 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... [2024/05/29 18:16:38.425 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/05/29 18:16:38.425 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] Starting Upstream TiDB... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 3 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 59-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6533640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24184, start at 2024-05-29 18:15:49.212230235 +0800 CST m=+4.212523024 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6534e0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:24265, start at 2024-05-29 18:15:49.315386073 +0800 CST m=+4.263348660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:49.321 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:49.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:49.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.28256.out cli tso query --pd=http://127.0.0.1:2379 Starting TLS TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67e714000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:35017, start at 2024-05-29 18:16:33.493622921 +0800 CST m=+4.370026770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:33.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:33.477 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:33.477 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... [2024/05/29 18:16:41.710 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/05/29 18:16:41.710 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] + set +x + tso='450095428857757699 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095428857757699 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:16:42 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.2830028302.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 60-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed [Wed May 29 18:16:42 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3666136663.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0005a8e10 TaskPositions:map[f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427431694337,"min-table-barrier-ts":450095427431694337,"admin-job-type":0} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0005a8e10 TaskPositions:map[f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427431694337,"min-table-barrier-ts":450095427431694337,"admin-job-type":0} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0005a8e10 TaskPositions:map[f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_+ grep -q 'etcd info' SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095427431694337,"min-table-barrier-ts":450095427431694337,"admin-job-type":0} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table ddl_manager.finish_mark exists check diff successfully "is-owner": true, run task successfully owner pid: 39229 owner id f533ff0c-9561-49e7-ab37-63ba8418d6fc [Wed May 29 18:16:44 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3937039372.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3885f215-f3fa-4a8a-bda5-12f46fc591ad {"id":"3885f215-f3fa-4a8a-bda5-12f46fc591ad","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9ef0191 3885f215-f3fa-4a8a-bda5-12f46fc591ad /tidb/cdc/default/default/upstream/7374363248142626349 {"id":7374363248142626349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3885f215-f3fa-4a8a-bda5-12f46fc591ad {"id":"3885f215-f3fa-4a8a-bda5-12f46fc591ad","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9ef0191 3885f215-f3fa-4a8a-bda5-12f46fc591ad /tidb/cdc/default/default/upstream/7374363248142626349 {"id":7374363248142626349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3885f215-f3fa-4a8a-bda5-12f46fc591ad {"id":"3885f215-f3fa-4a8a-bda5-12f46fc591ad","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9ef0191 3885f215-f3fa-4a8a-bda5-12f46fc591ad /tidb/cdc/default/default/upstream/7374363248142626349 {"id":7374363248142626349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.28359.out cli changefeed create --start-ts=450095428857757699 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 9f4d57f5-8432-4cf9-82b0-4fdc2b1fc10a Info: {"upstream_id":7374363248142626349,"namespace":"default","id":"9f4d57f5-8432-4cf9-82b0-4fdc2b1fc10a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:16:45.455931862+08:00","start_ts":450095428857757699,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095428857757699,"checkpoint_ts":450095428857757699,"checkpoint_time":"2024-05-29 18:16:41.734"} PASS wait process cdc.test exit for 1-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:45 GMT < Content-Length: 1049 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65dc6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29164, start at 2024-05-29 18:16:00.054802907 +0800 CST m=+4.536396844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:00.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:00.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:00.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f65db100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:29259, start at 2024-05-29 18:15:59.964148883 +0800 CST m=+4.365912722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:17:59.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:15:59.940 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:05:59.940 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:16:45 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.3338133383.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Wed May 29 18:16:45 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Wed May 29 18:16:47 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3673036732.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003526cd8 TaskPositions:map[12fd8a21-974c-455f-b05f-f9fa602096b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12fd8a21-974c-455f-b05f-f9fa602096b5 {"id":"12fd8a21-974c-455f-b05f-f9fa602096b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198720 12fd8a21-974c-455f-b05f-f9fa602096b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095430053134340,"min-table-barrier-ts":450095430315016196,"admin-job-type":0} /tidb/cdc/default/default/task/position/12fd8a21-974c-455f-b05f-f9fa602096b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003526cd8 TaskPositions:map[12fd8a21-974c-455f-b05f-f9fa602096b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12fd8a21-974c-455f-b05f-f9fa602096b5 {"id":"12fd8a21-974c-455f-b05f-f9fa602096b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198720 12fd8a21-974c-455f-b05f-f9fa602096b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095430053134340,"min-table-barrier-ts":450095430315016196,"admin-job-type":0} /tidb/cdc/default/default/task/position/12fd8a21-974c-455f-b05f-f9fa602096b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc003526cd8 TaskPositions:map[12fd8a21-974c-455f-b05f-f9fa602096b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f533ff0c-9561-49e7-ab37-63ba8418d6fc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12fd8a21-974c-455f-b05f-f9fa602096b5 {"id":"12fd8a21-974c-455f-b05f-f9fa602096b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f533ff0c-9561-49e7-ab37-63ba8418d6fc {"id":"f533ff0c-9561-49e7-ab37-63ba8418d6fc","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198700 f533ff0c-9561-49e7-ab37-63ba8418d6fc /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198720 12fd8a21-974c-455f-b05f-f9fa602096b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095430053134340,"min-table-barrier-ts":450095430315016196,"admin-job-type":0} /tidb/cdc/default/default/task/position/12fd8a21-974c-455f-b05f-f9fa602096b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f533ff0c-9561-49e7-ab37-63ba8418d6fc/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f533ff0c-9561-49e7-ab37-63ba8418d6fc" | grep id "cluster-id": "default" "id": "12fd8a21-974c-455f-b05f-f9fa602096b5", "cluster-id": "default" run task successfully capture_id: 12fd8a21-974c-455f-b05f-f9fa602096b5 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:48 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66332a4d-341f-4453-aa1f-f793a6e252b9 {"id":"66332a4d-341f-4453-aa1f-f793a6e252b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da067a33 66332a4d-341f-4453-aa1f-f793a6e252b9 /tidb/cdc/default/default/upstream/7374363291411941317 {"id":7374363291411941317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66332a4d-341f-4453-aa1f-f793a6e252b9 {"id":"66332a4d-341f-4453-aa1f-f793a6e252b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da067a33 66332a4d-341f-4453-aa1f-f793a6e252b9 /tidb/cdc/default/default/upstream/7374363291411941317 {"id":7374363291411941317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66332a4d-341f-4453-aa1f-f793a6e252b9 {"id":"66332a4d-341f-4453-aa1f-f793a6e252b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da067a33 66332a4d-341f-4453-aa1f-f793a6e252b9 /tidb/cdc/default/default/upstream/7374363291411941317 {"id":7374363291411941317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.33438.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: adcdc330-bb52-47a2-90e5-4bb447928a18 Info: {"upstream_id":7374363291411941317,"namespace":"default","id":"adcdc330-bb52-47a2-90e5-4bb447928a18","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:16:49.0036185+08:00","start_ts":450095430734970881,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095430734970881,"checkpoint_ts":450095430734970881,"checkpoint_time":"2024-05-29 18:16:48.895"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:16:50 GMT < Content-Length: 1303 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e359f88b-37dd-4b0f-a2bf-53e76e50f451 {"id":"e359f88b-37dd-4b0f-a2bf-53e76e50f451","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cd2 e359f88b-37dd-4b0f-a2bf-53e76e50f451 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e359f88b-37dd-4b0f-a2bf-53e76e50f451 {"id":"e359f88b-37dd-4b0f-a2bf-53e76e50f451","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cd2 e359f88b-37dd-4b0f-a2bf-53e76e50f451 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8699a95-f88a-431a-a211-359f8c19c2f2 {"id":"b8699a95-f88a-431a-a211-359f8c19c2f2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e359f88b-37dd-4b0f-a2bf-53e76e50f451 {"id":"e359f88b-37dd-4b0f-a2bf-53e76e50f451","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cad b8699a95-f88a-431a-a211-359f8c19c2f2 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fc3d9d20cd2 e359f88b-37dd-4b0f-a2bf-53e76e50f451 /tidb/cdc/default/default/upstream/7374363436739270955 {"id":7374363436739270955,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: check health nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later pass test: get status pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450095431923007492, "checkpoint_time": "2024-05-29 18:16:53.427", "error": null }' + echo '{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450095431923007492, "checkpoint_time": "2024-05-29 18:16:53.427", "error": null }' { "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450095431923007492, "checkpoint_time": "2024-05-29 18:16:53.427", "error": null } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 450095431923007492, '"checkpoint_time":' '"2024-05-29' '18:16:53.427",' '"error":' null '}' + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 450095431923007492, '"checkpoint_time":' '"2024-05-29' '18:16:53.427",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + info='{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450095431962591236, "checkpoint_time": "2024-05-29 18:16:53.578", "error": null }' + echo '{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450095431962591236, "checkpoint_time": "2024-05-29 18:16:53.578", "error": null }' { "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450095431962591236, "checkpoint_time": "2024-05-29 18:16:53.578", "error": null } ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 450095431962591236, '"checkpoint_time":' '"2024-05-29' '18:16:53.578",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 450095431962591236, '"checkpoint_time":' '"2024-05-29' '18:16:53.578",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450095431988805637, "checkpoint_time": "2024-05-29 18:16:53.678", "error": null }' + echo '{ "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450095431988805637, "checkpoint_time": "2024-05-29 18:16:53.678", "error": null }' { "upstream_id": 7374363436739270955, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450095431988805637, "checkpoint_time": "2024-05-29 18:16:53.678", "error": null } ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 450095431988805637, '"checkpoint_time":' '"2024-05-29' '18:16:53.678",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374363436739270955, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 450095431988805637, '"checkpoint_time":' '"2024-05-29' '18:16:53.678",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6780a40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30793, start at 2024-05-29 18:16:26.956996802 +0800 CST m=+4.397207003 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check_changefeed_status 127.0.0.1:8300 c240ec2d-fe8d-4034-924d-d746f75f53c3 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=c240ec2d-fe8d-4034-924d-d746f75f53c3 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/c240ec2d-fe8d-4034-924d-d746f75f53c3/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 765 0 --:--:-- --:--:-- --:--:-- 767 + info='{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' + echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' {"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045} ++ echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 c240ec2d-fe8d-4034-924d-d746f75f53c3 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=c240ec2d-fe8d-4034-924d-d746f75f53c3 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/c240ec2d-fe8d-4034-924d-d746f75f53c3/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1017 0 --:--:-- --:--:-- --:--:-- 1023 + info='{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' + echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' {"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045} ++ echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":450095432468267045,"checkpoint_ts":450095432468267045}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table sink_hang.t2 exists check diff failed 1-th time, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... check diff failed 4-th time, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d204000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30902, start at 2024-05-29 18:17:04.906511416 +0800 CST m=+4.322289898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:04.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.897 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.897 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 5-th time, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table changefeed_pause_resume.t1 not exists for 1-th check, retry later table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case delete_owner_key [Wed May 29 18:17:10 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3978539787.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 7-th time, retry later table changefeed_pause_resume.t3 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000ba18a8 TaskPositions:map[c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 80, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 82, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095436082446341,"min-table-barrier-ts":450095437130760198,"admin-job-type":0} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000ba18a8 TaskPositions:map[c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 80, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 82, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-bac+ grep -q 'failed to get info:' kend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095436082446341,"min-table-barrier-ts":450095437130760198,"admin-job-type":0} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc000ba18a8 TaskPositions:map[c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 80, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing tableID: 82, resolvedTs: 450095436082446341, checkpointTs: 450095436082446341, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095436082446341,"min-table-barrier-ts":450095437130760198,"admin-job-type":0} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully check diff failed 8-th time, retry later owner pid: 39790 owner id c80fcf98-68e4-40f5-b92e-92832a63ce3d default owner key /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 [Wed May 29 18:17:14 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.3995439956.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a56180017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30852, start at 2024-05-29 18:17:13.377654235 +0800 CST m=+4.297328370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log check diff failed 9-th time, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00083ef30 TaskPositions:map[9f84fb88-96aa-4788-9b44-0eadfe2766ad:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f84fb88-96aa-4788-9b44-0eadfe2766ad {"id":"9f84fb88-96aa-4788-9b44-0eadfe2766ad","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987ec 9f84fb88-96aa-4788-9b44-0eadfe2766ad /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095437917192203,"min-table-barrier-ts":450095437917192203,"admin-job-type":0} /tidb/cdc/default/default/task/position/9f84fb88-96aa-4788-9b44-0eadfe2766ad/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00083ef30 TaskPositions:map[9f84fb88-96aa-4788-9b44-0eadfe2766ad:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f84fb88-96aa-4788-9b44-0eadfe2766ad {"id":"9f84fb88-96aa-4788-9b44-0eadfe2766ad","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987ec 9f84fb88-96aa-4788-9b44-0eadfe2766ad /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval"+ grep -q 'failed to get info:' :200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095437917192203,"min-table-barrier-ts":450095437917192203,"admin-job-type":0} /tidb/cdc/default/default/task/position/9f84fb88-96aa-4788-9b44-0eadfe2766ad/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc00083ef30 TaskPositions:map[9f84fb88-96aa-4788-9b44-0eadfe2766ad:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c80fcf98-68e4-40f5-b92e-92832a63ce3d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f84fb88-96aa-4788-9b44-0eadfe2766ad {"id":"9f84fb88-96aa-4788-9b44-0eadfe2766ad","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c80fcf98-68e4-40f5-b92e-92832a63ce3d {"id":"c80fcf98-68e4-40f5-b92e-92832a63ce3d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987c9 c80fcf98-68e4-40f5-b92e-92832a63ce3d /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81987ec 9f84fb88-96aa-4788-9b44-0eadfe2766ad /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095437917192203,"min-table-barrier-ts":450095437917192203,"admin-job-type":0} /tidb/cdc/default/default/task/position/9f84fb88-96aa-4788-9b44-0eadfe2766ad/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c80fcf98-68e4-40f5-b92e-92832a63ce3d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "c80fcf98-68e4-40f5-b92e-92832a63ce3d default" | grep id "id": "9f84fb88-96aa-4788-9b44-0eadfe2766ad", run task successfully capture_id: 9f84fb88-96aa-4788-9b44-0eadfe2766ad 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '9f84fb88-96aa-4788-9b44-0eadfe2766ad' 9f84fb88-96aa-4788-9b44-0eadfe2766ad run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 177 100 177 0 0 166k 0 --:--:-- --:--:-- --:--:-- 172k "is_owner": false run task successfully check diff failed 10-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 11-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 4-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.simple1 exists pass test: list changefeed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f67801c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30701, start at 2024-05-29 18:16:26.917888628 +0800 CST m=+4.405689252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6780a40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:30793, start at 2024-05-29 18:16:26.956996802 +0800 CST m=+4.397207003 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:18:26.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:16:26.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:06:26.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:17:21 CST 2024] <<<<<< START cdc server in new_ci_collation_with_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_with_old_value.3478734789.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: get changefeed pass test: pause changefeed pass test: update changefeed check diff failed 12-th time, retry later pass test: resume changefeed check diff failed 5-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later pass test: rebalance table pass test: list processors [2024/05/29 18:17:23.019 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 12.449738094s"] pass test: get processors pass test: move table pass test: set log level check diff failed 13-th time, retry later pass test: remove changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3f696c-d9c7-4114-b253-9db112a48e6e {"id":"5e3f696c-d9c7-4114-b253-9db112a48e6e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da8c188e 5e3f696c-d9c7-4114-b253-9db112a48e6e /tidb/cdc/default/default/upstream/7374363402226331912 {"id":7374363402226331912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3f696c-d9c7-4114-b253-9db112a48e6e {"id":"5e3f696c-d9c7-4114-b253-9db112a48e6e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da8c188e 5e3f696c-d9c7-4114-b253-9db112a48e6e /tidb/cdc/default/default/upstream/7374363402226331912 {"id":7374363402226331912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3f696c-d9c7-4114-b253-9db112a48e6e {"id":"5e3f696c-d9c7-4114-b253-9db112a48e6e","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3da8c188e 5e3f696c-d9c7-4114-b253-9db112a48e6e /tidb/cdc/default/default/upstream/7374363402226331912 {"id":7374363402226331912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: eabacca8-aa3a-4f77-83f3-a0d63eccd7ab Info: {"upstream_id":7374363402226331912,"namespace":"default","id":"eabacca8-aa3a-4f77-83f3-a0d63eccd7ab","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-29T18:17:24.510385638+08:00","start_ts":450095439216640001,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095439216640001,"checkpoint_ts":450095439216640001,"checkpoint_time":"2024-05-29 18:17:21.250"} pass test: resign owner pass test: get tso pass test: verify table pass test: create changefeed v2 status code 200 pass test: show metadata status code 204 status code 204 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock ***************** properties ***************** "scanproportion"="0" "recordcount"="10" "threadcount"="2" "readallfields"="true" "insertproportion"="0" "updateproportion"="0" "workload"="core" "mysql.user"="root" "mysql.db"="multi_capture_1" "operationcount"="0" "readproportion"="0" "mysql.port"="4000" "dotransactions"="false" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 9.536214ms INSERT - Takes(s): 0.0, Count: 9, OPS: 1751.2, Avg(us): 1540, Min(us): 1055, Max(us): 4205, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.port"="4000" "scanproportion"="0" "dotransactions"="false" "mysql.user"="root" "workload"="core" "recordcount"="10" "readproportion"="0" "mysql.db"="multi_capture_2" "operationcount"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.037609ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2120.0, Avg(us): 1735, Min(us): 1015, Max(us): 4283, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readproportion"="0" "workload"="core" "mysql.user"="root" "operationcount"="0" "requestdistribution"="uniform" "recordcount"="10" "dotransactions"="false" "readallfields"="true" "mysql.db"="multi_capture_3" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "updateproportion"="0" "scanproportion"="0" "threadcount"="2" ********************************************** Run finished, takes 10.936294ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1938.3, Avg(us): 2066, Min(us): 1059, Max(us): 5605, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "recordcount"="10" "mysql.host"="127.0.0.1" "threadcount"="2" "readallfields"="true" "mysql.port"="4000" "readproportion"="0" "updateproportion"="0" "dotransactions"="false" "insertproportion"="0" "mysql.db"="multi_capture_4" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "scanproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 9.779629ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1829.0, Avg(us): 1837, Min(us): 1011, Max(us): 4184, 95th(us): 5000, 99th(us): 5000 [Wed May 29 18:17:20 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3652936531.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:23 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:17:23 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3659536597.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:17:26 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> check diff failed 14-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:26 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:17:26 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3665636658.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table partition_table.t exists table partition_table.t1 not exists for 1-th check, retry later check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t1 not exists for 2-th check, retry later check diff failed 15-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:29 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/14f1b691-c214-48bf-b92f-8def0306c8b0 {"id":"14f1b691-c214-48bf-b92f-8def0306c8b0","address":"127.0.0.1:8303","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef90 14f1b691-c214-48bf-b92f-8def0306c8b0 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/14f1b691-c214-48bf-b92f-8def0306c8b0 {"id":"14f1b691-c214-48bf-b92f-8def0306c8b0","address":"127.0.0.1:8303","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef90 14f1b691-c214-48bf-b92f-8def0306c8b0 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f2452d1-21d8-4ffc-83e2-6d957dbfd850 {"id":"0f2452d1-21d8-4ffc-83e2-6d957dbfd850","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/14f1b691-c214-48bf-b92f-8def0306c8b0 {"id":"14f1b691-c214-48bf-b92f-8def0306c8b0","address":"127.0.0.1:8303","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/a86cfd08-d759-49a6-bc71-27ffb9174d56 {"id":"a86cfd08-d759-49a6-bc71-27ffb9174d56","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef5e 0f2452d1-21d8-4ffc-83e2-6d957dbfd850 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef7f a86cfd08-d759-49a6-bc71-27ffb9174d56 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d998ef90 14f1b691-c214-48bf-b92f-8def0306c8b0 /tidb/cdc/default/default/upstream/7374363173860617352 {"id":7374363173860617352,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.36713.out cli changefeed create --start-ts=450095422674567169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: f6949a6c-9363-414c-937b-d521f2513370 Info: {"upstream_id":7374363173860617352,"namespace":"default","id":"f6949a6c-9363-414c-937b-d521f2513370","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:17:29.69443038+08:00","start_ts":450095422674567169,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095422674567169,"checkpoint_ts":450095422674567169,"checkpoint_time":"2024-05-29 18:16:18.147"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table multi_capture_1.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d204000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30902, start at 2024-05-29 18:17:04.906511416 +0800 CST m=+4.322289898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:04.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.897 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.897 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:17:31 CST 2024] <<<<<< run test case sink_hang success! >>>>>> check diff failed 3-th time, retry later table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later table partition_table.t2 not exists for 2-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 4-th time, retry later ***************** properties ***************** "operationcount"="0" "readallfields"="true" "mysql.db"="consistent_replicate_storage_s3" "fieldcount"="100" "workload"="core" "threadcount"="10" "mysql.user"="root" "mysql.port"="4000" "insertproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "recordcount"="5000" "mysql.host"="127.0.0.1" "updateproportion"="0" "readproportion"="0" "dotransactions"="false" ********************************************** table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later table partition_table.t2 not exists for 3-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.106.78 + export UP_TIDB_HOST=10.233.106.78 UP_PD_HOST_1=10.233.106.78 UP_PD_HOST_2=10.233.106.78 UP_PD_HOST_3=10.233.106.78 UP_TIKV_HOST_1=10.233.106.78 UP_TIKV_HOST_2=10.233.106.78 UP_TIKV_HOST_3=10.233.106.78 + UP_TIDB_HOST=10.233.106.78 + UP_PD_HOST_1=10.233.106.78 + UP_PD_HOST_2=10.233.106.78 + UP_PD_HOST_3=10.233.106.78 + UP_TIKV_HOST_1=10.233.106.78 + UP_TIKV_HOST_2=10.233.106.78 + UP_TIKV_HOST_3=10.233.106.78 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=18065 + trap 'stop_tidb_cluster && stop_proxy' EXIT + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.106.78' UP_TIDB_HOST 10.233.106.78 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.106.78' UP_PD_HOST_1 10.233.106.78 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.106.78' UP_PD_HOST_2 10.233.106.78 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.106.78' UP_PD_HOST_3 10.233.106.78 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.106.78' UP_TIKV_HOST_1 10.233.106.78 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.106.78' UP_TIKV_HOST_2 10.233.106.78 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.106.78' UP_TIKV_HOST_3 10.233.106.78 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster Run finished, takes 3.068205185s INSERT - Takes(s): 3.1, Count: 5000, OPS: 1634.5, Avg(us): 5770, Min(us): 1593, Max(us): 76586, 95th(us): 44000, 99th(us): 62000 table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later table partition_table.t2 not exists for 4-th check, retry later table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 6-th time, retry later table multi_capture_4.usertable exists check diff failed 1-th time, retry later table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5738 0 --:--:-- --:--:-- --:--:-- 5666 100 221 100 221 0 0 5723 0 --:--:-- --:--:-- --:--:-- 5666 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:17:26.166","puller_resolved_ts":"2024-05-29 18:17:26.166","last_synced_ts":"2024-05-29 18:15:10.016","now_ts":"2024-05-29 18:17:27.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:17:26.166","puller_resolved_ts":"2024-05-29' '18:17:26.166","last_synced_ts":"2024-05-29' '18:15:10.016","now_ts":"2024-05-29' '18:17:27.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 24380 6.7 0.0 12824764 150876 ? Sl 18:14 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 24430 3.8 0.0 12651368 138820 ? Sl 18:14 0:07 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-29T18:17:34.863+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:34.866+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:34.940+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a56180017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30852, start at 2024-05-29 18:17:13.377654235 +0800 CST m=+4.297328370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully ***************** properties ***************** "insertproportion"="0" "operationcount"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "workload"="core" "mysql.db"="multi_capture_1" "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "scanproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 11.184256ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1989.9, Avg(us): 946, Min(us): 430, Max(us): 1584, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "mysql.port"="4000" "recordcount"="20" "mysql.db"="multi_capture_2" "readallfields"="true" "requestdistribution"="uniform" "scanproportion"="0" "threadcount"="2" "operationcount"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.user"="root" "readproportion"="0" "insertproportion"="0" "workload"="core" ********************************************** Run finished, takes 10.505432ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2104.9, Avg(us): 900, Min(us): 501, Max(us): 1388, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "insertproportion"="0" "updateproportion"="0" "operationcount"="0" "recordcount"="20" "scanproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "mysql.host"="127.0.0.1" "workload"="core" "dotransactions"="false" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="multi_capture_3" "readallfields"="true" ********************************************** Run finished, takes 9.125306ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2480.8, Avg(us): 845, Min(us): 462, Max(us): 1371, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "readproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="20" "updateproportion"="0" "dotransactions"="false" "operationcount"="0" "mysql.db"="multi_capture_4" "scanproportion"="0" "threadcount"="2" "workload"="core" "readallfields"="true" "requestdistribution"="uniform" "mysql.user"="root" "insertproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 10.566437ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2141.1, Avg(us): 987, Min(us): 574, Max(us): 1609, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 2-th check, retry later {"level":"warn","ts":"2024-05-29T18:17:40.865+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:40.867+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:40.943+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later [2024/05/29 18:17:40.471 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`binlog_big`] [where="((TRUE) AND TRUE)"] ["source checksum"=214543849] ["target checksum"=0] ["get source checksum cost"=26.113744ms] ["get target checksum cost"=1.238936ms] [2024/05/29 18:17:40.941 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`binlog_big`] [where="((TRUE) AND TRUE)"] [cost=469.777174ms] [2024/05/29 18:17:40.944 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] check diff failed 1-th time, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table drop_tables.c not exists for 1-th check, retry later table drop_tables.c not exists for 2-th check, retry later table drop_tables.c not exists for 3-th check, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... table partition_table.finish_mark exists check diff successfully table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:17:43 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:17:43 CST 2024] <<<<<< run test case partition_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_kill_capture [Wed May 29 18:17:44 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4041240414.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table drop_tables.c not exists for 4-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... {"level":"warn","ts":"2024-05-29T18:17:44.847+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:17:44.847+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-29T18:17:44.853+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:17:44.853+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-29T18:17:44.928+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:17:44.928+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716977864.9400349,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014341c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1716977864.9400861,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716977864.9867046,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0015d56c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1716977864.9867594,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716977865.037018,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0018c8a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1716977865.0370934,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release split_and_random_merge scale: 20 check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later Verifying downstream PD is started... {"level":"warn","ts":"2024-05-29T18:17:46.865+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:46.869+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:46.944+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed *************************** 1. row *************************** count(distinct region_id): 7 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0009d8f48 TaskPositions:map[e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating tableID: 82, resolvedTs: 450095445794619397, checkpointTs: 450095445794619397, state: Replicating tableID: 84, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095445794619397,"min-table-barrier-ts":450095445794619397,"admin-job-type":0} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0009d8f48 TaskPositions:map[e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating tableID: 82, resolvedTs: 450095445794619397, checkpointTs: 450095445794619397, state: Replicating tableID: 84, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095445794619397,"min-table-barrier-ts":450095445794619397,"admin-job-type":0} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0009d8f48 TaskPositions:map[e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating tableID: 82, resolvedTs: 450095445794619397, checkpointTs: 450095445794619397, state: Replicating tableID: 84, resolvedTs: 450095445820833797, checkpointTs: 450095445794619397, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095445794619397,"min-table-barrier-ts":450095445794619397,"admin-job-type":0} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully check diff failed 4-th time, retry later owner pid: 40417 owner id e735f191-ab89-4aab-a3e4-3367e801bbba table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table drop_tables.c exists Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d370000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30807, start at 2024-05-29 18:17:05.001627183 +0800 CST m=+4.501698893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:05.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f69d204000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--5kn82, pid:30902, start at 2024-05-29 18:17:04.906511416 +0800 CST m=+4.322289898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:04.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:04.897 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:04.897 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:17:48 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3497934981.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc0 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later check diff failed 5-th time, retry later check diff successfully nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Wed May 29 18:17:49 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4057340575.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:17:50 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.106.78' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:17:51 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3504835050.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) split_and_random_merge scale: 40 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:17:52.867+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:52.871+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:52.946+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0006c7170 TaskPositions:map[ade99457-3df1-4b91-9576-e2d65f7e142e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade99457-3df1-4b91-9576-e2d65f7e142e {"id":"ade99457-3df1-4b91-9576-e2d65f7e142e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988f4 ade99457-3df1-4b91-9576-e2d65f7e142e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095447367745545,"min-table-barrier-ts":450095447367745545,"admin-job-type":0} /tidb/cdc/default/default/task/position/ade99457-3df1-4b91-9576-e2d65f7e142e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0006c7170 TaskPositions:map[ade99457-3df1-4b91-9576-e2d65f7e142e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade99457-3df1-4b91-9576-e2d65f7e142e {"id":"ade99457-3df1-4b91-9576-e2d65f7e142e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988f4 ade99457-3df1-4b91-9576-e2d65f7e142e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095447367745545,"min-table-barrier-ts":450095447367745545,"admin-job-type":0} /tidb/cdc/default/default/task/position/ade99457-3df1-4b91-9576-e2d65f7e142e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0006c7170 TaskPositions:map[ade99457-3df1-4b91-9576-e2d65f7e142e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e735f191-ab89-4aab-a3e4-3367e801bbba:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ade99457-3df1-4b91-9576-e2d65f7e142e {"id":"ade99457-3df1-4b91-9576-e2d65f7e142e","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e735f191-ab89-4aab-a3e4-3367e801bbba {"id":"e735f191-ab89-4aab-a3e4-3367e801bbba","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988c0 e735f191-ab89-4aab-a3e4-3367e801bbba /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81988f4 ade99457-3df1-4b91-9576-e2d65f7e142e /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095447367745545,"min-table-barrier-ts":450095447367745545,"admin-job-type":0} /tidb/cdc/default/default/task/position/ade99457-3df1-4b91-9576-e2d65f7e142e/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e735f191-ab89-4aab-a3e4-3367e801bbba/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "e735f191-ab89-4aab-a3e4-3367e801bbba" | grep id "cluster-id": "default" "id": "ade99457-3df1-4b91-9576-e2d65f7e142e", "cluster-id": "default" run task successfully table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.106.78' (111) check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc6a40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37217, start at 2024-05-29 18:17:53.327943765 +0800 CST m=+4.723874655 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.106.78' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:54 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:17:54 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3510135103.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce6080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38130, start at 2024-05-29 18:17:55.351942269 +0800 CST m=+4.303485561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:17:58 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 {"id":"4cf25dbb-cf2f-4e38-859e-8cd18ec5d576","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdc3 4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 {"id":"4cf25dbb-cf2f-4e38-859e-8cd18ec5d576","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdc3 4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cf409cf-fcbc-45d4-bba8-2198101773b5 {"id":"0cf409cf-fcbc-45d4-bba8-2198101773b5","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 {"id":"4cf25dbb-cf2f-4e38-859e-8cd18ec5d576","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b {"id":"e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cd8e 0cf409cf-fcbc-45d4-bba8-2198101773b5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdb1 e95bdf8c-5ad9-41f9-b3a2-c6f29fc6963b /tidb/cdc/default/__cdc_meta__/owner/22318fc3daf9cdc3 4cf25dbb-cf2f-4e38-859e-8cd18ec5d576 /tidb/cdc/default/default/upstream/7374363574310834660 {"id":7374363574310834660,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.cli.35164.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a53040024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30756, start at 2024-05-29 18:17:13.196827355 +0800 CST m=+4.167718995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6a56180017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:30852, start at 2024-05-29 18:17:13.377654235 +0800 CST m=+4.297328370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:13.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:13.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:13.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:17:58 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3485534857.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Create changefeed successfully! ID: 331b252e-b1c4-42ca-b82d-c248337f1683 Info: {"upstream_id":7374363574310834660,"namespace":"default","id":"331b252e-b1c4-42ca-b82d-c248337f1683","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-29T18:17:58.41558363+08:00","start_ts":450095448924094479,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095448924094479,"checkpoint_ts":450095448924094479,"checkpoint_time":"2024-05-29 18:17:58.281"} PASS check diff failed 3-th time, retry later Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:17:58.868+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x {"level":"warn","ts":"2024-05-29T18:17:58.871+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:17:58.948+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/29 18:17:58.448 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later Starting CDC state checker... nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00190a34-616c-461c-a8df-f561d3c92dea {"id":"00190a34-616c-461c-a8df-f561d3c92dea","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230c8e 00190a34-616c-461c-a8df-f561d3c92dea /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00190a34-616c-461c-a8df-f561d3c92dea {"id":"00190a34-616c-461c-a8df-f561d3c92dea","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230c8e 00190a34-616c-461c-a8df-f561d3c92dea /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00190a34-616c-461c-a8df-f561d3c92dea {"id":"00190a34-616c-461c-a8df-f561d3c92dea","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230c8e 00190a34-616c-461c-a8df-f561d3c92dea /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.34929.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7374363604926208491,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-29T18:18:01.72325644+08:00","start_ts":450095449805422593,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095449805422593,"checkpoint_ts":450095449805422593,"checkpoint_time":"2024-05-29 18:18:01.643"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:18:04.870+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:04.872+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:04.948+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_capture [Wed May 29 18:18:06 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4078640788.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37912, start at 2024-05-29 18:18:04.997059317 +0800 CST m=+4.326191981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 2-th time, retry later Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... check diff failed 3-th time, retry later Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002edbc20 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating tableID: 82, resolvedTs: 450095451562049542, checkpointTs: 450095451562049542, state: Replicating tableID: 84, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095451562049542,"min-table-barrier-ts":450095451562049542,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002edbc20 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating tableID: 82, resolvedTs: 450095451562049542, checkpointTs: 450095451562049542, state: Replicating tableID: 84, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095451562049542,"min-table-barrier-ts":450095451562049542,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002edbc20 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating tableID: 82, resolvedTs: 450095451562049542, checkpointTs: 450095451562049542, state: Replicating tableID: 84, resolvedTs: 450095451601371139, checkpointTs: 450095451562049542, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095451562049542,"min-table-barrier-ts":450095451562049542,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later "is-owner": true, run task successfully # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... owner pid: 40791 owner id d7033eda-007d-4ede-a523-1a90b2ef8968 [Wed May 29 18:18:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4092840930.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad60000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36860, start at 2024-05-29 18:18:08.097619895 +0800 CST m=+4.245669218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.104 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/29 18:18:05.052 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:18:05.053 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:18:05.053 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095418561527812] [resolvedTs=450095450045022214] [2024/05/29 18:18:05.054 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:18:05.054 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:18:05.054 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=1.08086ms] [2024/05/29 18:18:05.054 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.085816ms] [2024/05/29 18:18:05.058 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:18:05.060 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:18:05.062 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:18:05.062 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:18:05.062 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:18:05.065 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:18:05.067 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:18:05.067 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450095418954743816,\"CommitTs\":450095418954743820,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/29 18:18:05.067 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:05.268 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:05.469 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:05.669 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:05.870 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:06.071 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:06.271 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:06.472 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:06.672 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:06.873 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.073 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450095418954743816,\"CommitTs\":450095418954743820,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/29 18:18:07.297 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="alter table t1 drop partition p1"] [duration=224.092531ms] [namespace=] [changefeed=] [2024/05/29 18:18:07.297 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450095418994065416,\"CommitTs\":450095418994065421,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.297 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.497 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.497 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450095418994065416,\"CommitTs\":450095418994065421,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/29 18:18:07.648 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [duration=150.441336ms] [namespace=] [changefeed=] [2024/05/29 18:18:07.648 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450095419033124873,\"CommitTs\":450095419033124878,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.648 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:07.849 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:08.050 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:08.050 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:08.050 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/29 18:18:08.050 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450095419033124873,\"CommitTs\":450095419033124878,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/29 18:18:08.195 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [duration=145.564947ms] [namespace=] [changefeed=] [2024/05/29 18:18:08.195 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.195 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.296 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.296 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.396 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.396 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.496 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.496 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.597 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.597 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.697 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.697 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.797 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.797 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.898 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=450095450045022214] [2024/05/29 18:18:08.898 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.200 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.200 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.500 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.500 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.601 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.601 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.702 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.702 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=450095450045022214] [2024/05/29 18:18:09.803 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=47] [appliedDDLCount=3] [currentCheckpoint=450095450045022214] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:18:09.803 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] Apply redo log successfully check diff successfully [Wed May 29 18:18:09 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 test 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:18:10.872+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:10.874+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:10.950+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff failed 4-th time, retry later table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002fca978 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f8181b9b-0bc4-4d50-ae05-40ea6c7125b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 {"id":"f8181b9b-0bc4-4d50-ae05-40ea6c7125b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198972 f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095452610625548,"min-table-barrier-ts":450095452610625548,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002fca978 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f8181b9b-0bc4-4d50-ae05-40ea6c7125b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 {"id":"f8181b9b-0bc4-4d50-ae05-40ea6c7125b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198972 f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval"+ grep -q 'failed to get info:' :200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095452610625548,"min-table-barrier-ts":450095452610625548,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc002fca978 TaskPositions:map[d7033eda-007d-4ede-a523-1a90b2ef8968:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f8181b9b-0bc4-4d50-ae05-40ea6c7125b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7033eda-007d-4ede-a523-1a90b2ef8968 {"id":"d7033eda-007d-4ede-a523-1a90b2ef8968","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 {"id":"f8181b9b-0bc4-4d50-ae05-40ea6c7125b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198947 d7033eda-007d-4ede-a523-1a90b2ef8968 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198972 f8181b9b-0bc4-4d50-ae05-40ea6c7125b5 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095452610625548,"min-table-barrier-ts":450095452610625548,"admin-job-type":0} /tidb/cdc/default/default/task/position/d7033eda-007d-4ede-a523-1a90b2ef8968/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8181b9b-0bc4-4d50-ae05-40ea6c7125b5/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "d7033eda-007d-4ede-a523-1a90b2ef8968" | grep id "cluster-id": "default" "id": "f8181b9b-0bc4-4d50-ae05-40ea6c7125b5", "cluster-id": "default" run task successfully table consistent_replicate_storage_s3.t2 exists nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:18:14 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3566835670.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc6a40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37217, start at 2024-05-29 18:17:53.327943765 +0800 CST m=+4.723874655 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:18:16.873+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e28700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:16.876+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000560700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:18:16.951+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0009de000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' [2024/05/29 18:18:17.139 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e588525b-6ca4-437a-9730-8e7f0a7ae99f {ClusterID:default ID:default/e588525b-6ca4-437a-9730-8e7f0a7ae99f Info:{"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} Status:0xc00122b830 TaskPositions:map[76cae9ff-ca0b-47af-bddc-f79a0005934c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 94, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 87, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 82, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 84, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 88, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 89, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 91, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76cae9ff-ca0b-47af-bddc-f79a0005934c {"id":"76cae9ff-ca0b-47af-bddc-f79a0005934c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72d6f 76cae9ff-ca0b-47af-bddc-f79a0005934c /tidb/cdc/default/default/changefeed/info/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} /tidb/cdc/default/default/changefeed/status/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":450095453590257668,"min-table-barrier-ts":450095453616472066,"admin-job-type":0} /tidb/cdc/default/default/task/position/76cae9ff-ca0b-47af-bddc-f79a0005934c/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e588525b-6ca4-437a-9730-8e7f0a7ae99f {ClusterID:default ID:default/e588525b-6ca4-437a-9730-8e7f0a7ae99f Info:{"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} Status:0xc00122b830 TaskPositions:map[76cae9ff-ca0b-47af-bddc-f79a0005934c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 94, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 87, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 82, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 84, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 88, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 89, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 91, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76cae9ff-ca0b-47af-bddc-f79a0005934c {"id":"76cae9ff-ca0b-47af-bddc-f79a0005934c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72d6f 76cae9ff-ca0b-47af-bddc-f79a0005934c /tidb/cdc/default/default/changefeed/info/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} /tidb/cdc/default/default/changefeed/status/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":450095453590257668,"min-table-barrier-ts":450095453616472066,"admin-job-type":0} /tidb/cdc/default/default/task/position/76cae9ff-ca0b-47af-bddc-f79a0005934c/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e588525b-6ca4-437a-9730-8e7f0a7ae99f {ClusterID:default ID:default/e588525b-6ca4-437a-9730-8e7f0a7ae99f Info:{"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} Status:0xc00122b830 TaskPositions:map[76cae9ff-ca0b-47af-bddc-f79a0005934c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 94, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 87, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 82, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 84, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 88, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 89, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating tableID: 91, resolvedTs: 450095453590257669, checkpointTs: 450095453590257668, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76cae9ff-ca0b-47af-bddc-f79a0005934c {"id":"76cae9ff-ca0b-47af-bddc-f79a0005934c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d9d72d6f 76cae9ff-ca0b-47af-bddc-f79a0005934c /tidb/cdc/default/default/changefeed/info/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"upstream-id":7374363255541117893,"namespace":"default","changefeed-id":"e588525b-6ca4-437a-9730-8e7f0a7ae99f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:16:38.207632956+08:00","start-ts":450095427900145667,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095427926360068} /tidb/cdc/default/default/changefeed/status/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":450095453590257668,"min-table-barrier-ts":450095453616472066,"admin-job-type":0} /tidb/cdc/default/default/task/position/76cae9ff-ca0b-47af-bddc-f79a0005934c/e588525b-6ca4-437a-9730-8e7f0a7ae99f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363255541117893 {"id":7374363255541117893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:18:17 CST 2024] <<<<<< run test case cdc success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster check diff failed 2-th time, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35f8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29440, start at 2024-05-29 18:18:16.841834038 +0800 CST m=+4.203102001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.830 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.830 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting CDC state checker... check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.106.78' (111) check diff failed 4-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce6080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38130, start at 2024-05-29 18:17:55.351942269 +0800 CST m=+4.303485561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37912, start at 2024-05-29 18:18:04.997059317 +0800 CST m=+4.326191981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table new_ci_collation_with_old_value_test.t1 exists table new_ci_collation_with_old_value_test.t2 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t2 exists table new_ci_collation_with_old_value_test.t3 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t3 exists table new_ci_collation_with_old_value_test.t4 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t4 exists table new_ci_collation_with_old_value_test.t5 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t5 exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later check diff successfully check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f20f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35988, start at 2024-05-29 18:18:31.917123405 +0800 CST m=+4.315433619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:31.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:31.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:31.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_capture [Wed May 29 18:18:34 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4116541167.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:18:35 CST 2024] <<<<<< run test case new_ci_collation_with_old_value success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc4f00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37124, start at 2024-05-29 18:17:53.259259841 +0800 CST m=+4.717887663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.262 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.262 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6cc6a40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:37217, start at 2024-05-29 18:17:53.327943765 +0800 CST m=+4.723874655 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:53.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:53.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:53.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41307.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0010643c0 TaskPositions:map[5205d130-3add-4bf0-8bc3-1cedea017b9d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 84, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 80, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5205d130-3add-4bf0-8bc3-1cedea017b9d {"id":"5205d130-3add-4bf0-8bc3-1cedea017b9d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81989e8 5205d130-3add-4bf0-8bc3-1cedea017b9d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095457944993798,"min-table-barrier-ts":450095458915188740,"admin-job-type":0} /tidb/cdc/default/default/task/position/5205d130-3add-4bf0-8bc3-1cedea017b9d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0010643c0 TaskPositions:map[5205d130-3add-4bf0-8bc3-1cedea017b9d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 84, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 80, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5205d130-3add-4bf0-8bc3-1cedea017b9d {"id":"5205d130-3add-4bf0-8bc3-1cedea017b9d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81989e8 5205d130-3add-4bf0-8bc3-1cedea017b9d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095457944993798,"min-table-barrier-ts":450095458915188740,"admin-job-type":0} /tidb/cdc/default/default/task/position/5205d130-3add-4bf0-8bc3-1cedea017b9d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0010643c0 TaskPositions:map[5205d130-3add-4bf0-8bc3-1cedea017b9d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 84, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing tableID: 80, resolvedTs: 450095457944993798, checkpointTs: 450095457944993798, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5205d130-3add-4bf0-8bc3-1cedea017b9d {"id":"5205d130-3add-4bf0-8bc3-1cedea017b9d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81989e8 5205d130-3add-4bf0-8bc3-1cedea017b9d /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095457944993798,"min-table-barrier-ts":450095458915188740,"admin-job-type":0} /tidb/cdc/default/default/task/position/5205d130-3add-4bf0-8bc3-1cedea017b9d/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x + tso='450095459098165249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095459098165249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x "is-owner": true, run task successfully owner pid: 41170 owner id 5205d130-3add-4bf0-8bc3-1cedea017b9d process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '5205d130-3add-4bf0-8bc3-1cedea017b9d' /tidb/cdc/default/__cdc_meta__/owner/22318fc3d81989e8 run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad60000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36860, start at 2024-05-29 18:18:08.097619895 +0800 CST m=+4.245669218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.104 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f6ef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26682, start at 2024-05-29 18:18:36.900721759 +0800 CST m=+4.315105766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log check diff failed 6-th time, retry later check_redo_resolved_ts e588525b-6ca4-437a-9730-8e7f0a7ae99f 450095459528081412 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/e588525b-6ca4-437a-9730-8e7f0a7ae99f/meta global resolved ts 450095458846244870 not forward to 450095459528081412 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Wed May 29 18:18:40 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4137941381.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts e588525b-6ca4-437a-9730-8e7f0a7ae99f 450095459528081412 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/e588525b-6ca4-437a-9730-8e7f0a7ae99f/meta run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/29 18:18:41.555 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:18:41.557 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:18:41.560 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:18:41.560 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095453996580876] [resolvedTs=450095459894820870] [2024/05/29 18:18:41.561 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:18:41.561 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:18:41.561 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:18:41.561 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:18:41.567 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:18:41.567 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=7.147524ms] [2024/05/29 18:18:41.573 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:18:41.575 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:18:41.575 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:18:41.575 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:18:41.583 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:18:41.586 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [2024/05/29 18:18:42.176 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=615.714072ms] [2024/05/29 18:18:42.177 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":450095453996580871,\"CommitTs\":450095453996580876,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_s3\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 1-th time, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fa9840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37410, start at 2024-05-29 18:18:40.637920368 +0800 CST m=+4.279710762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.609 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.609 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log [2024/05/29 18:18:43.181 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095459894820870] [2024/05/29 18:18:43.181 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450095459894820870] [2024/05/29 18:18:43.282 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095459894820870] [2024/05/29 18:18:43.282 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450095459894820870] Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:18:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0018949f0 TaskPositions:map[33b26829-0ee0-4a13-b458-062c01389ba0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 80, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 82, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b26829-0ee0-4a13-b458-062c01389ba0 {"id":"33b26829-0ee0-4a13-b458-062c01389ba0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198a29 33b26829-0ee0-4a13-b458-062c01389ba0 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095459701620741,"min-table-barrier-ts":450095460579803138,"admin-job-type":0} /tidb/cdc/default/default/task/position/33b26829-0ee0-4a13-b458-062c01389ba0/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0018949f0 TaskPositions:map[33b26829-0ee0-4a13-b458-062c01389ba0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 80, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 82, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b26829-0ee0-4a13-b458-062c01389ba0 {"id":"33b26829-0ee0-4a13-b458-062c01389ba0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198a29 33b26829-0ee0-4a13-b458-062c01389ba0 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095459701620741,"min-table-barrier-ts":450095460579803138,"admin-job-type":0} /tidb/cdc/default/default/task/position/33b26829-0ee0-4a13-b458-062c01389ba0/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/33146874-74af-4ab6-a7c9-794a2ee8c8dc {ClusterID:default ID:default/33146874-74af-4ab6-a7c9-794a2ee8c8dc Info:{"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} Status:0xc0018949f0 TaskPositions:map[33b26829-0ee0-4a13-b458-062c01389ba0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 80, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing tableID: 82, resolvedTs: 450095459701620741, checkpointTs: 450095459701620741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b26829-0ee0-4a13-b458-062c01389ba0 {"id":"33b26829-0ee0-4a13-b458-062c01389ba0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3d8198a29 33b26829-0ee0-4a13-b458-062c01389ba0 /tidb/cdc/default/default/changefeed/info/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"upstream-id":7374362733485705949,"namespace":"default","changefeed-id":"33146874-74af-4ab6-a7c9-794a2ee8c8dc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:15:42.826594229+08:00","start-ts":450095396957454337,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095413406728197} /tidb/cdc/default/default/changefeed/status/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":450095459701620741,"min-table-barrier-ts":450095460579803138,"admin-job-type":0} /tidb/cdc/default/default/task/position/33b26829-0ee0-4a13-b458-062c01389ba0/33146874-74af-4ab6-a7c9-794a2ee8c8dc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374362733485705949 {"id":7374362733485705949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully [2024/05/29 18:18:43.985 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450095459894820870] [2024/05/29 18:18:43.985 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450095459894820870] [2024/05/29 18:18:44.086 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=450095459894820870] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:18:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] Apply redo log successfully owner pid: 41384 owner id 33b26829-0ee0-4a13-b458-062c01389ba0 changefeed id: 33146874-74af-4ab6-a7c9-794a2ee8c8dc curl -s -d "cf-id=33146874-74af-4ab6-a7c9-794a2ee8c8dc&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully [Wed May 29 18:18:44 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 796 1229 --:--:-- --:--:-- --:--:-- 1255 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35f8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29440, start at 2024-05-29 18:18:16.841834038 +0800 CST m=+4.203102001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.830 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.830 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_without_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.106.78' (111) start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce48c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38036, start at 2024-05-29 18:17:55.272844292 +0800 CST m=+4.272767734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6ce6080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2dh97, pid:38130, start at 2024-05-29 18:17:55.351942269 +0800 CST m=+4.303485561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:19:55.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:17:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:07:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=42220 + echo 'proxy port: 18065' proxy port: 18065 + echo 'proxy pid: 42220' proxy pid: 42220 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=18065 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f check diff failed 4-th time, retry later time="2024-05-29T18:18:49+08:00" level=info msg="Listening on 127.0.0.1:18065" time="2024-05-29T18:18:49+08:00" level=info msg="Not intercepting TLS connections" nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:18:51 CST 2024] <<<<<< run test case availability success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + export http_proxy=http://127.0.0.1:18065 + http_proxy=http://127.0.0.1:18065 + export https_proxy=http://127.0.0.1:18065 + https_proxy=http://127.0.0.1:18065 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.106.78:2379' try to connect pd cluster via proxy, pd addr: 10.233.106.78:2379 + ensure 10 curl http://10.233.106.78:2379/ curl http://10.233.106.78:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 16351 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 42220 started proxy pid: 42220 + echo started proxy at port: 18065 started proxy at port: 18065 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.106.78 2379 + pd_host=10.233.106.78 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://10.233.106.78:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.42787.out cli tso query --pd=http://10.233.106.78:2379 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... failed proxying s2c rpc error: code = Unavailable desc = client disconnected + set +x + tso='450095463590002691 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095463590002691 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095463590002691 + echo 'query start ts: 450095463590002691' query start ts: 450095463590002691 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Wed May 29 18:18:54 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.4282942831.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 18065 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 18065 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Wed, 29 May 2024 10:18:54 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 1-th time, retry later failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + workdir=/tmp/tidb_cdc_test/canal_json_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_partition_table ++ date + echo '[Wed May 29 18:18:53 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>>' [Wed May 29 18:18:53 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37825, start at 2024-05-29 18:18:04.999894212 +0800 CST m=+4.382653491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6d7cc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:37912, start at 2024-05-29 18:18:04.997059317 +0800 CST m=+4.326191981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:05.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:04.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:04.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.41987.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095464068939777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095464068939777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f20f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35988, start at 2024-05-29 18:18:31.917123405 +0800 CST m=+4.315433619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:31.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:31.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:31.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 18065 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 18065 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 616 < Content-Type: text/plain; charset=utf-8 < Date: Wed, 29 May 2024 10:18:57 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2eace419-45f4-4bf9-9cdb-1f947d854131 {"id":"2eace419-45f4-4bf9-9cdb-1f947d854131","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3e28fc3dbe10693 2eace419-45f4-4bf9-9cdb-1f947d854131 /tidb/cdc/default/default/upstream/7374363780262910332 {"id":7374363780262910332,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.106.78:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2eace419-45f4-4bf9-9cdb-1f947d854131 {"id":"2eace419-45f4-4bf9-9cdb-1f947d854131","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3e28fc3dbe10693 2eace419-45f4-4bf9-9cdb-1f947d854131 /tidb/cdc/default/default/upstream/7374363780262910332 {"id":7374363780262910332,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.106.78:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2eace419-45f4-4bf9-9cdb-1f947d854131 {"id":"2eace419-45f4-4bf9-9cdb-1f947d854131","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3e28fc3dbe10693 2eace419-45f4-4bf9-9cdb-1f947d854131 /tidb/cdc/default/default/upstream/7374363780262910332 {"id":7374363780262910332,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.106.78:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=450095463590002691 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.42904.out cli changefeed create --start-ts=450095463590002691 --sink-uri=blackhole:/// ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) failed proxying s2c rpc error: code = Canceled desc = context canceled Create changefeed successfully! ID: 7734dda9-b4dc-4c97-8b0a-0eb607e91994 Info: {"upstream_id":7374363780262910332,"namespace":"default","id":"7734dda9-b4dc-4c97-8b0a-0eb607e91994","sink_uri":"blackhole:///","create_time":"2024-05-29T18:18:57.863911486+08:00","start_ts":450095463590002691,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095463590002691,"checkpoint_ts":450095463590002691,"checkpoint_time":"2024-05-29 18:18:54.227"} PASS table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad68001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36773, start at 2024-05-29 18:18:08.139853533 +0800 CST m=+4.355700498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6dad60000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:36860, start at 2024-05-29 18:18:08.097619895 +0800 CST m=+4.245669218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:08.104 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:08.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:08.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b3500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36151, start at 2024-05-29 18:18:57.650853723 +0800 CST m=+4.233663722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table.t not exists for 1-th check, retry later Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later check diff failed 5-th time, retry later + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq + service_type_count=6 + echo 'captured services: ' captured services: + echo . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv '[2024/05' . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv [2024/05 + '[' 6 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Wed May 29 18:19:03 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Wed May 29 18:19:03 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected table partition_table.t not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29349, start at 2024-05-29 18:18:16.803346222 +0800 CST m=+4.247659387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6e35f8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:29440, start at 2024-05-29 18:18:16.841834038 +0800 CST m=+4.203102001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:16.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:16.830 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:16.830 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:19:03 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.3358133583.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_only_block_related_table.finish_mark exists /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (34860) - No such process wait process 34860 exit for 1-th time... process 34860 already exit [Wed May 29 18:19:05 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3506935071.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t exists table partition_table.t1 exists check diff successfully table partition_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53c3691-56bd-4248-a297-e62f6f765ec3 {"id":"d53c3691-56bd-4248-a297-e62f6f765ec3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc1f378e d53c3691-56bd-4248-a297-e62f6f765ec3 /tidb/cdc/default/default/upstream/7374363872981479220 {"id":7374363872981479220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53c3691-56bd-4248-a297-e62f6f765ec3 {"id":"d53c3691-56bd-4248-a297-e62f6f765ec3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc1f378e d53c3691-56bd-4248-a297-e62f6f765ec3 /tidb/cdc/default/default/upstream/7374363872981479220 {"id":7374363872981479220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53c3691-56bd-4248-a297-e62f6f765ec3 {"id":"d53c3691-56bd-4248-a297-e62f6f765ec3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc1f378e d53c3691-56bd-4248-a297-e62f6f765ec3 /tidb/cdc/default/default/upstream/7374363872981479220 {"id":7374363872981479220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.33639.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: 8013d27c-afd1-44af-a356-d9f4ab5fab23 Info: {"upstream_id":7374363872981479220,"namespace":"default","id":"8013d27c-afd1-44af-a356-d9f4ab5fab23","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:19:07.336472542+08:00","start_ts":450095467002855428,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095467002855428,"checkpoint_ts":450095467002855428,"checkpoint_time":"2024-05-29 18:19:07.246"} PASS table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7139240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36971, start at 2024-05-29 18:19:06.223282048 +0800 CST m=+4.544307742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc002e135c0 TaskPositions:map[cb842b66-648c-4f2c-a7b4-fc82237842f9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 88, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 82, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 84, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb842b66-648c-4f2c-a7b4-fc82237842f9 {"id":"cb842b66-648c-4f2c-a7b4-fc82237842f9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230dc0 cb842b66-648c-4f2c-a7b4-fc82237842f9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467067342854,"min-table-barrier-ts":450095467067342854,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb842b66-648c-4f2c-a7b4-fc82237842f9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc002e135c0 TaskPositions:map[cb842b66-648c-4f2c-a7b4-fc82237842f9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 88, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 82, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 84, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb842b66-648c-4f2c-a7b4-fc82237842f9 {"id":"cb842b66-648c-4f2c-a7b4-fc82237842f9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230dc0 cb842b66-648c-4f2c-a7b4-fc82237842f9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467067342854,"min-table-barrier-ts":450095467067342854,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb842b66-648c-4f2c-a7b4-fc82237842f9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc002e135c0 TaskPositions:map[cb842b66-648c-4f2c-a7b4-fc82237842f9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 88, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 82, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating tableID: 84, resolvedTs: 450095467067342856, checkpointTs: 450095467067342854, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb842b66-648c-4f2c-a7b4-fc82237842f9 {"id":"cb842b66-648c-4f2c-a7b4-fc82237842f9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230dc0 cb842b66-648c-4f2c-a7b4-fc82237842f9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compressio+ grep -q 'etcd info' n":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467067342854,"min-table-barrier-ts":450095467067342854,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb842b66-648c-4f2c-a7b4-fc82237842f9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f6ef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26682, start at 2024-05-29 18:18:36.900721759 +0800 CST m=+4.315105766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check diff failed 1-th time, retry later check diff failed 2-th time, retry later table partition_table.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fa9840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37410, start at 2024-05-29 18:18:40.637920368 +0800 CST m=+4.279710762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.609 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.609 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... check diff successfully check_ts_not_forward ddl-only-block-related-table # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 3-th time, retry later table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f1c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42518, start at 2024-05-29 18:19:11.727963074 +0800 CST m=+4.149904247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.737 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.737 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log table partition_table.finish_mark not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 5-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f235c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35894, start at 2024-05-29 18:18:32.046164108 +0800 CST m=+4.492922439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:32.052 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:32.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:32.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f20f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:35988, start at 2024-05-29 18:18:31.917123405 +0800 CST m=+4.315433619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:31.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:31.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:31.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:19:15 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.3987739879.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + stop_proxy + kill 42220 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:19:14 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Starting CDC state checker... table partition_table.finish_mark not exists for 6-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:19:17 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> table partition_table.finish_mark not exists for 7-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b5e63ae-18d4-400b-b52b-a023a9b08dcf {"id":"6b5e63ae-18d4-400b-b52b-a023a9b08dcf","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6d8e 6b5e63ae-18d4-400b-b52b-a023a9b08dcf /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b5e63ae-18d4-400b-b52b-a023a9b08dcf {"id":"6b5e63ae-18d4-400b-b52b-a023a9b08dcf","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6d8e 6b5e63ae-18d4-400b-b52b-a023a9b08dcf /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b5e63ae-18d4-400b-b52b-a023a9b08dcf {"id":"6b5e63ae-18d4-400b-b52b-a023a9b08dcf","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6d8e 6b5e63ae-18d4-400b-b52b-a023a9b08dcf /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table partition_table.finish_mark not exists for 8-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table.finish_mark not exists for 9-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b3500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36151, start at 2024-05-29 18:18:57.650853723 +0800 CST m=+4.233663722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table partition_table.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 11-th check, retry later Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 12-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f70900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26588, start at 2024-05-29 18:18:36.968971291 +0800 CST m=+4.478901870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6f6ef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:26682, start at 2024-05-29 18:18:36.900721759 +0800 CST m=+4.315105766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:36.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:36.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:36.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30611.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095472664117249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095472664117249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095472664117249 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:19:29 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3063630638.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 13-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fabe00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37320, start at 2024-05-29 18:18:40.80428222 +0800 CST m=+4.504961058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.812 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f6fa9840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:37410, start at 2024-05-29 18:18:40.637920368 +0800 CST m=+4.279710762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:40.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:40.609 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:40.609 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.41405.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... + set +x + tso='450095473072013313 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095473072013313 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7139240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36971, start at 2024-05-29 18:19:06.223282048 +0800 CST m=+4.544307742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table partition_table.finish_mark not exists for 14-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfdadc0d-6bb9-4074-be88-ab29f1d160e4 {"id":"cfdadc0d-6bb9-4074-be88-ab29f1d160e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc7bfa8e cfdadc0d-6bb9-4074-be88-ab29f1d160e4 /tidb/cdc/default/default/upstream/7374363966616185874 {"id":7374363966616185874,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfdadc0d-6bb9-4074-be88-ab29f1d160e4 {"id":"cfdadc0d-6bb9-4074-be88-ab29f1d160e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc7bfa8e cfdadc0d-6bb9-4074-be88-ab29f1d160e4 /tidb/cdc/default/default/upstream/7374363966616185874 {"id":7374363966616185874,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfdadc0d-6bb9-4074-be88-ab29f1d160e4 {"id":"cfdadc0d-6bb9-4074-be88-ab29f1d160e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc7bfa8e cfdadc0d-6bb9-4074-be88-ab29f1d160e4 /tidb/cdc/default/default/upstream/7374363966616185874 {"id":7374363966616185874,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095472664117249 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30706.out cli changefeed create --start-ts=450095472664117249 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374363966616185874,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:19:32.568466069+08:00","start_ts":450095472664117249,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095472664117249,"checkpoint_ts":450095472664117249,"checkpoint_time":"2024-05-29 18:19:28.842"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 15-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:19:36 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f1c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42518, start at 2024-05-29 18:19:11.727963074 +0800 CST m=+4.149904247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.737 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.737 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b4d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36049, start at 2024-05-29 18:18:57.749841967 +0800 CST m=+4.437769425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f70b3500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--4t2d9, pid:36151, start at 2024-05-29 18:18:57.650853723 +0800 CST m=+4.233663722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:20:57.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:18:57.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:08:57.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:19:42 CST 2024] <<<<<< START cdc server in new_ci_collation_without_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_without_old_value.4018340185.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7371c00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39791, start at 2024-05-29 18:19:42.593241876 +0800 CST m=+4.460890530 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log ***************** properties ***************** "operationcount"="0" "workload"="core" "mysql.port"="4000" "readallfields"="true" "updateproportion"="0" "threadcount"="2" "dotransactions"="false" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "insertproportion"="0" "scanproportion"="0" "recordcount"="10" "mysql.db"="sink_retry" "requestdistribution"="uniform" ********************************************** Run finished, takes 11.037497ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1641.4, Avg(us): 2120, Min(us): 1213, Max(us): 4907, 95th(us): 5000, 99th(us): 5000 [Wed May 29 18:19:35 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.4144341445.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4639e5ca-88b0-4a1c-af26-0ebd6a87d89a {"id":"4639e5ca-88b0-4a1c-af26-0ebd6a87d89a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dbb19e19 4639e5ca-88b0-4a1c-af26-0ebd6a87d89a /tidb/cdc/default/default/upstream/7374363781912893930 {"id":7374363781912893930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4639e5ca-88b0-4a1c-af26-0ebd6a87d89a {"id":"4639e5ca-88b0-4a1c-af26-0ebd6a87d89a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dbb19e19 4639e5ca-88b0-4a1c-af26-0ebd6a87d89a /tidb/cdc/default/default/upstream/7374363781912893930 {"id":7374363781912893930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4639e5ca-88b0-4a1c-af26-0ebd6a87d89a {"id":"4639e5ca-88b0-4a1c-af26-0ebd6a87d89a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dbb19e19 4639e5ca-88b0-4a1c-af26-0ebd6a87d89a /tidb/cdc/default/default/upstream/7374363781912893930 {"id":7374363781912893930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41504.out cli changefeed create --start-ts=450095459098165249 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 3595bff8-ba9a-4f4b-833e-d77f6c786586 Info: {"upstream_id":7374363781912893930,"namespace":"default","id":"3595bff8-ba9a-4f4b-833e-d77f6c786586","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:19:38.435194114+08:00","start_ts":450095459098165249,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095459098165249,"checkpoint_ts":450095459098165249,"checkpoint_time":"2024-05-29 18:18:37.092"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5829fe95-c5cb-4225-88be-24bd5ce75ef4 {"id":"5829fe95-c5cb-4225-88be-24bd5ce75ef4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcb2ad91 5829fe95-c5cb-4225-88be-24bd5ce75ef4 /tidb/cdc/default/default/upstream/7374364058743517085 {"id":7374364058743517085,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5829fe95-c5cb-4225-88be-24bd5ce75ef4 {"id":"5829fe95-c5cb-4225-88be-24bd5ce75ef4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcb2ad91 5829fe95-c5cb-4225-88be-24bd5ce75ef4 /tidb/cdc/default/default/upstream/7374364058743517085 {"id":7374364058743517085,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5829fe95-c5cb-4225-88be-24bd5ce75ef4 {"id":"5829fe95-c5cb-4225-88be-24bd5ce75ef4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcb2ad91 5829fe95-c5cb-4225-88be-24bd5ce75ef4 /tidb/cdc/default/default/upstream/7374364058743517085 {"id":7374364058743517085,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 2f5222d2-d068-48b2-a2cb-d22de12f5d7a Info: {"upstream_id":7374364058743517085,"namespace":"default","id":"2f5222d2-d068-48b2-a2cb-d22de12f5d7a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-29T18:19:45.973995969+08:00","start_ts":450095476296122369,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095476296122369,"checkpoint_ts":450095476296122369,"checkpoint_time":"2024-05-29 18:19:42.697"} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7136d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36866, start at 2024-05-29 18:19:06.044395635 +0800 CST m=+4.465921013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7139240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:36971, start at 2024-05-29 18:19:06.223282048 +0800 CST m=+4.544307742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:06.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:06.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:06.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:19:49 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.4113341135.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/005d17b8-0b12-44ef-aee1-869e3f064c33 {"id":"005d17b8-0b12-44ef-aee1-869e3f064c33","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcd2358e 005d17b8-0b12-44ef-aee1-869e3f064c33 /tidb/cdc/default/default/upstream/7374364093043695115 {"id":7374364093043695115,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/005d17b8-0b12-44ef-aee1-869e3f064c33 {"id":"005d17b8-0b12-44ef-aee1-869e3f064c33","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcd2358e 005d17b8-0b12-44ef-aee1-869e3f064c33 /tidb/cdc/default/default/upstream/7374364093043695115 {"id":7374364093043695115,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/005d17b8-0b12-44ef-aee1-869e3f064c33 {"id":"005d17b8-0b12-44ef-aee1-869e3f064c33","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcd2358e 005d17b8-0b12-44ef-aee1-869e3f064c33 /tidb/cdc/default/default/upstream/7374364093043695115 {"id":7374364093043695115,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.41196.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: 3c62e867-9560-4cc1-87d0-e6183b8f3304 Info: {"upstream_id":7374364093043695115,"namespace":"default","id":"3c62e867-9560-4cc1-87d0-e6183b8f3304","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-05-29T18:19:53.155232665+08:00","start_ts":450095479015342083,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095479015342083,"checkpoint_ts":450095479015342083,"checkpoint_time":"2024-05-29 18:19:53.070"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42418, start at 2024-05-29 18:19:11.703766406 +0800 CST m=+4.177694883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f718f1c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:42518, start at 2024-05-29 18:19:11.727963074 +0800 CST m=+4.149904247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:11.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:11.737 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:11.737 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later [Wed May 29 18:19:54 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.4209642098.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:19:57 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/d0ef0931-7473-4d5c-a1fb-5278e9be9956 {"id":"d0ef0931-7473-4d5c-a1fb-5278e9be9956","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fc3dbfda819 d0ef0931-7473-4d5c-a1fb-5278e9be9956 /tidb/cdc/test1/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/d0ef0931-7473-4d5c-a1fb-5278e9be9956 {"id":"d0ef0931-7473-4d5c-a1fb-5278e9be9956","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fc3dbfda819 d0ef0931-7473-4d5c-a1fb-5278e9be9956 /tidb/cdc/test1/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/d0ef0931-7473-4d5c-a1fb-5278e9be9956 {"id":"d0ef0931-7473-4d5c-a1fb-5278e9be9956","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fc3dbfda819 d0ef0931-7473-4d5c-a1fb-5278e9be9956 /tidb/cdc/test1/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:19:57 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.4215842160.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:01 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/1aa643a3-8349-41ac-a304-38efd1157ef0 {"id":"1aa643a3-8349-41ac-a304-38efd1157ef0","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fc3dbfda82f 1aa643a3-8349-41ac-a304-38efd1157ef0 /tidb/cdc/test2/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/1aa643a3-8349-41ac-a304-38efd1157ef0 {"id":"1aa643a3-8349-41ac-a304-38efd1157ef0","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fc3dbfda82f 1aa643a3-8349-41ac-a304-38efd1157ef0 /tidb/cdc/test2/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/1aa643a3-8349-41ac-a304-38efd1157ef0 {"id":"1aa643a3-8349-41ac-a304-38efd1157ef0","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fc3dbfda82f 1aa643a3-8349-41ac-a304-38efd1157ef0 /tidb/cdc/test2/default/upstream/7374363831131660443 {"id":7374363831131660443,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.42212.out cli changefeed create --start-ts=450095464068939777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: 994182c2-3357-4081-a6de-9898a3ec2ed6 Info: {"upstream_id":7374363831131660443,"namespace":"default","id":"994182c2-3357-4081-a6de-9898a3ec2ed6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:20:01.385058034+08:00","start_ts":450095464068939777,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095464068939777,"checkpoint_ts":450095464068939777,"checkpoint_time":"2024-05-29 18:18:56.054"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.42252.out cli changefeed create --start-ts=450095464068939777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: fa8c5344-5069-44b3-bf9d-3cbf017110ee Info: {"upstream_id":7374363831131660443,"namespace":"default","id":"fa8c5344-5069-44b3-bf9d-3cbf017110ee","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:20:02.017683457+08:00","start_ts":450095464068939777,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095464068939777,"checkpoint_ts":450095464068939777,"checkpoint_time":"2024-05-29 18:18:56.054"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.multi_cdc1 not exists for 1-th check, retry later table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later [Wed May 29 18:20:03 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.4108541087.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688f9a1-3737-400d-b073-7ab8705a4e35 {"id":"5688f9a1-3737-400d-b073-7ab8705a4e35","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc079e7f 5688f9a1-3737-400d-b073-7ab8705a4e35 /tidb/cdc/default/default/upstream/7374363843116986056 {"id":7374363843116986056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688f9a1-3737-400d-b073-7ab8705a4e35 {"id":"5688f9a1-3737-400d-b073-7ab8705a4e35","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc079e7f 5688f9a1-3737-400d-b073-7ab8705a4e35 /tidb/cdc/default/default/upstream/7374363843116986056 {"id":7374363843116986056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5688f9a1-3737-400d-b073-7ab8705a4e35 {"id":"5688f9a1-3737-400d-b073-7ab8705a4e35","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc079e7f 5688f9a1-3737-400d-b073-7ab8705a4e35 /tidb/cdc/default/default/upstream/7374363843116986056 {"id":7374363843116986056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 6187bc49-19a1-4ae9-82f4-2df470b57088 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6187bc49-19a1-4ae9-82f4-2df470b57088 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6187bc49-19a1-4ae9-82f4-2df470b57088 -s table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later + info='{ "upstream_id": 7374363843116986056, "namespace": "default", "id": "6187bc49-19a1-4ae9-82f4-2df470b57088", "state": "normal", "checkpoint_tso": 450095482524401668, "checkpoint_time": "2024-05-29 18:20:06.456", "error": null }' + echo '{ "upstream_id": 7374363843116986056, "namespace": "default", "id": "6187bc49-19a1-4ae9-82f4-2df470b57088", "state": "normal", "checkpoint_tso": 450095482524401668, "checkpoint_time": "2024-05-29 18:20:06.456", "error": null }' { "upstream_id": 7374363843116986056, "namespace": "default", "id": "6187bc49-19a1-4ae9-82f4-2df470b57088", "state": "normal", "checkpoint_tso": 450095482524401668, "checkpoint_time": "2024-05-29 18:20:06.456", "error": null } ++ echo '{' '"upstream_id":' 7374363843116986056, '"namespace":' '"default",' '"id":' '"6187bc49-19a1-4ae9-82f4-2df470b57088",' '"state":' '"normal",' '"checkpoint_tso":' 450095482524401668, '"checkpoint_time":' '"2024-05-29' '18:20:06.456",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374363843116986056, '"namespace":' '"default",' '"id":' '"6187bc49-19a1-4ae9-82f4-2df470b57088",' '"state":' '"normal",' '"checkpoint_tso":' 450095482524401668, '"checkpoint_time":' '"2024-05-29' '18:20:06.456",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:07 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table event_filter.t_alter exists table event_filter.finish_mark exists table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "readproportion"="0" "operationcount"="0" "workload"="core" "dotransactions"="false" "recordcount"="10" "mysql.host"="127.0.0.1" "scanproportion"="0" "updateproportion"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="sink_retry" "readallfields"="true" "insertproportion"="0" "threadcount"="2" ********************************************** Run finished, takes 4.395519ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3284.1, Avg(us): 776, Min(us): 518, Max(us): 1643, 95th(us): 2000, 99th(us): 2000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7371c00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39791, start at 2024-05-29 18:19:42.593241876 +0800 CST m=+4.460890530 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff successfully ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:13 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> check diff failed 1-th time, retry later ***************** properties ***************** "readproportion"="0" "mysql.user"="root" "mysql.db"="consistent_replicate_ddl" "fieldcount"="3" "scanproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "recordcount"="50" "threadcount"="10" "operationcount"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "workload"="core" "dotransactions"="false" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 9.756156ms INSERT - Takes(s): 0.0, Count: 49, OPS: 10582.7, Avg(us): 1735, Min(us): 820, Max(us): 5250, 95th(us): 6000, 99th(us): 6000 table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:16 CST 2024] <<<<<< run test case event_filter success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_changefeed/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later Verifying downstream PD is started... table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable_bak exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764d240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43334, start at 2024-05-29 18:20:29.41124979 +0800 CST m=+4.336873512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/error.log Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 29751 7.2 0.3 3168568 1454660 ? Sl 18:19 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 29752 6.8 0.3 3163448 1442244 ? Sl 18:19 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 29753 6.7 0.3 3154744 1442544 ? Sl 18:19 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 29755 6.7 0.3 3158840 1443460 ? Sl 18:19 0:04 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2473 0 --:--:-- --:--:-- --:--:-- 2479 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:20:30.741","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-29 18:20:26.841","now_ts":"2024-05-29 18:20:31.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:20:30.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:20:26.841","now_ts":"2024-05-29' '18:20:31.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:20:30.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:20:26.841","now_ts":"2024-05-29' '18:20:31.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f736e000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39692, start at 2024-05-29 18:19:42.371651066 +0800 CST m=+4.338211710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7371c00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:39791, start at 2024-05-29 18:19:42.593241876 +0800 CST m=+4.460890530 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:21:42.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:19:42.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:09:42.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:20:32 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.4380343805.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2404ab12-6e9f-450c-92d4-c3f972310552 {"id":"2404ab12-6e9f-450c-92d4-c3f972310552","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fb8f 2404ab12-6e9f-450c-92d4-c3f972310552 /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2404ab12-6e9f-450c-92d4-c3f972310552 {"id":"2404ab12-6e9f-450c-92d4-c3f972310552","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fb8f 2404ab12-6e9f-450c-92d4-c3f972310552 /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2404ab12-6e9f-450c-92d4-c3f972310552 {"id":"2404ab12-6e9f-450c-92d4-c3f972310552","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fb8f 2404ab12-6e9f-450c-92d4-c3f972310552 /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:35 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.4017340175.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) try a VALID cdc server command [Wed May 29 18:20:28 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.4152241524.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afd656fd-8181-4667-ad0d-d4d61ca6e6ef {"id":"afd656fd-8181-4667-ad0d-d4d61ca6e6ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc839a10 afd656fd-8181-4667-ad0d-d4d61ca6e6ef /tidb/cdc/default/default/upstream/7374363985788729991 {"id":7374363985788729991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afd656fd-8181-4667-ad0d-d4d61ca6e6ef {"id":"afd656fd-8181-4667-ad0d-d4d61ca6e6ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc839a10 afd656fd-8181-4667-ad0d-d4d61ca6e6ef /tidb/cdc/default/default/upstream/7374363985788729991 {"id":7374363985788729991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afd656fd-8181-4667-ad0d-d4d61ca6e6ef {"id":"afd656fd-8181-4667-ad0d-d4d61ca6e6ef","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc839a10 afd656fd-8181-4667-ad0d-d4d61ca6e6ef /tidb/cdc/default/default/upstream/7374363985788729991 {"id":7374363985788729991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/7ecb4849-52e6-4d57-936c-6e803a6bb8df {ClusterID:default ID:default/7ecb4849-52e6-4d57-936c-6e803a6bb8df Info:{"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} Status:0xc0001ccca8 TaskPositions:map[e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 88, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 90, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 86, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 82, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 {"id":"e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6f26 e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 /tidb/cdc/default/default/changefeed/info/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} /tidb/cdc/default/default/changefeed/status/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":450095490720333831,"min-table-barrier-ts":450095490720333831,"admin-job-type":0} /tidb/cdc/default/default/task/position/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7ecb4849-52e6-4d57-936c-6e803a6bb8df {ClusterID:default ID:default/7ecb4849-52e6-4d57-936c-6e803a6bb8df Info:{"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} Status:0xc0001ccca8 TaskPositions:map[e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 88, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 90, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 86, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 82, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 {"id":"e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6f26 e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 /tidb/cdc/default/default/changefeed/info/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} /tidb/cdc/default/default/changefeed/status/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":450095490720333831,"min-table-barrier-ts":450095490720333831,"admin-job-type":0} /tidb/cdc/default/default/task/position/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7ecb4849-52e6-4d57-936c-6e803a6bb8df {ClusterID:default ID:default/7ecb4849-52e6-4d57-936c-6e803a6bb8df Info:{"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} Status:0xc0001ccca8 TaskPositions:map[e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 88, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 90, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 86, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating tableID: 82, resolvedTs: 450095490720333832, checkpointTs: 450095490720333831, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 {"id":"e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dc4c6f26 e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336 /tidb/cdc/default/default/changefeed/info/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"upstream-id":7374363948355654855,"namespace":"default","changefeed-id":"7ecb4849-52e6-4d57-936c-6e803a6bb8df","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:19:18.705477618+08:00","start-ts":450095469984743427,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095469997850633} /tidb/cdc/default/default/changefeed/status/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":450095490720333831,"min-table-barrier-ts":450095490720333831,"admin-job-type":0} /tidb/cdc/default/default/task/position/e0fb30d7-59c4-4f5c-b4db-ffb0c0c1d336/7ecb4849-52e6-4d57-936c-6e803a6bb8df {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363948355654855 {"id":7374363948355654855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f250001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34916, start at 2024-05-29 18:20:40.002195674 +0800 CST m=+4.163935634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log table new_ci_collation_without_old_value_test.t1 exists table new_ci_collation_without_old_value_test.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f8240004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42396, start at 2024-05-29 18:20:40.332492413 +0800 CST m=+4.278516844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log table sink_retry.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:39 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table new_ci_collation_without_old_value_test.t2 exists table new_ci_collation_without_old_value_test.t3 not exists for 1-th check, retry later Starting CDC state checker... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table new_ci_collation_without_old_value_test.t3 exists table new_ci_collation_without_old_value_test.t4 not exists for 1-th check, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 35074 exit for 1-th time... wait process 35074 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (35074) - No such process wait process 35074 exit for 3-th time... process 35074 already exit [Wed May 29 18:20:35 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3585135853.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:20:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc00151f080 TaskPositions:map[fa31572b-0540-40b1-9e16-d23a67deb40a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 82, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 84, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 86, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa31572b-0540-40b1-9e16-d23a67deb40a {"id":"fa31572b-0540-40b1-9e16-d23a67deb40a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230ea8 fa31572b-0540-40b1-9e16-d23a67deb40a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467329749011,"min-table-barrier-ts":450095490686779396,"admin-job-type":0} /tidb/cdc/default/default/task/position/fa31572b-0540-40b1-9e16-d23a67deb40a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc00151f080 TaskPositions:map[fa31572b-0540-40b1-9e16-d23a67deb40a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 82, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 84, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 86, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa31572b-0540-40b1-9e16-d23a67deb40a {"id":"fa31572b-0540-40b1-9e16-d23a67deb40a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230ea8 fa31572b-0540-40b1-9e16-d23a67deb40a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467329749011,"min-table-barrier-ts":450095490686779396,"admin-job-type":0} /tidb/cdc/default/default/task/position/fa31572b-0540-40b1-9e16-d23a67deb40a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} Status:0xc00151f080 TaskPositions:map[fa31572b-0540-40b1-9e16-d23a67deb40a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 82, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 84, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing tableID: 86, resolvedTs: 450095467329749011, checkpointTs: 450095467329749011, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa31572b-0540-40b1-9e16-d23a67deb40a {"id":"fa31572b-0540-40b1-9e16-d23a67deb40a","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3db230ea8 fa31572b-0540-40b1-9e16-d23a67deb40a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374363604926208491,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-29T18:18:01.72325644+08:00","start-ts":450095449805422593,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095449818267653} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450095467329749011,"min-table-barrier-ts":450095490686779396,"admin-job-type":0} /tidb/cdc/default/default/task/position/fa31572b-0540-40b1-9e16-d23a67deb40a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374363604926208491 {"id":7374363604926208491,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 450095467329749000->450095491211067400 checkpoint: 450095467329749000->450095491211067400 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:41 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> table new_ci_collation_without_old_value_test.t4 exists table new_ci_collation_without_old_value_test.t5 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sorter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Wed May 29 18:20:49 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Wed May 29 18:20:49 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml table new_ci_collation_without_old_value_test.t5 exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Wed May 29 18:20:51 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.4160441606.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None + [[ true != \n\o ]] + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:20:55 CST 2024] <<<<<< run test case new_ci_collation_without_old_value success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764d240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43334, start at 2024-05-29 18:20:29.41124979 +0800 CST m=+4.336873512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully [Wed May 29 18:21:01 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7831740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42669, start at 2024-05-29 18:21:00.427272662 +0800 CST m=+4.330040415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sorter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sorter/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Wed May 29 18:21:05 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f786aa00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:37030, start at 2024-05-29 18:21:04.054661065 +0800 CST m=+4.132344461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:04.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:04.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log Verifying downstream PD is started... [Wed May 29 18:20:57 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.4673246734.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:21:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bebd71d9-ee0d-4a0a-919b-595974912319 {"id":"bebd71d9-ee0d-4a0a-919b-595974912319","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcf68b0f bebd71d9-ee0d-4a0a-919b-595974912319 /tidb/cdc/default/default/upstream/7374364114199004675 {"id":7374364114199004675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bebd71d9-ee0d-4a0a-919b-595974912319 {"id":"bebd71d9-ee0d-4a0a-919b-595974912319","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcf68b0f bebd71d9-ee0d-4a0a-919b-595974912319 /tidb/cdc/default/default/upstream/7374364114199004675 {"id":7374364114199004675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bebd71d9-ee0d-4a0a-919b-595974912319 {"id":"bebd71d9-ee0d-4a0a-919b-595974912319","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dcf68b0f bebd71d9-ee0d-4a0a-919b-595974912319 /tidb/cdc/default/default/upstream/7374364114199004675 {"id":7374364114199004675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.46789.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 205b649b-4777-4af1-b554-7d314c5956b1 Info: {"upstream_id":7374364114199004675,"namespace":"default","id":"205b649b-4777-4af1-b554-7d314c5956b1","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-29T18:21:00.669267309+08:00","start_ts":450095496709537793,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095496709537793,"checkpoint_ts":450095496709537793,"checkpoint_time":"2024-05-29 18:21:00.568"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 [2024/05/29 18:21:01.440 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:01.441 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:01.441 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq0] [2024/05/29 18:21:01.955 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:01.963 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.655609ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:01.963 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq1] [2024/05/29 18:21:02.472 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:02.480 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.751651ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:02.480 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq2] [2024/05/29 18:21:02.989 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:02.997 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.240862ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:02.997 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq3] [2024/05/29 18:21:03.506 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:03.513 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.647946ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:03.514 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq4] [2024/05/29 18:21:04.022 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:04.030 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.504607ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/29 18:21:04.030 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts0] [2024/05/29 18:21:04.545 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.189697ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/29 18:21:04.546 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts1] [2024/05/29 18:21:05.062 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.157384ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/29 18:21:05.063 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts2] [2024/05/29 18:21:05.580 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.069068ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/29 18:21:05.580 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts3] [2024/05/29 18:21:06.098 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.465207ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/29 18:21:06.098 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts4] [2024/05/29 18:21:06.614 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.772463ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/29 18:21:09.124 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f8240004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42396, start at 2024-05-29 18:20:40.332492413 +0800 CST m=+4.278516844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:21:11.134 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f250001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34916, start at 2024-05-29 18:20:40.002195674 +0800 CST m=+4.163935634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:21:13.144 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Wed May 29 18:21:11 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2024/05/29 18:21:15.153 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:21:17.163 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764b980019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43245, start at 2024-05-29 18:20:29.324165461 +0800 CST m=+4.302701744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f764d240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--8m7c3, pid:43334, start at 2024-05-29 18:20:29.41124979 +0800 CST m=+4.336873512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:29.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:29.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:29.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:21:18 CST 2024] <<<<<< START cdc server in multi_changefeed case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_changefeed.4742447426.out server --log-file /tmp/tidb_cdc_test/multi_changefeed/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_changefeed/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:21:19.171 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:21.180 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:21:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e841e-eed2-401d-9583-fef592c977c4 {"id":"505e841e-eed2-401d-9583-fef592c977c4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de2ff48e 505e841e-eed2-401d-9583-fef592c977c4 /tidb/cdc/default/default/upstream/7374364445975694966 {"id":7374364445975694966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e841e-eed2-401d-9583-fef592c977c4 {"id":"505e841e-eed2-401d-9583-fef592c977c4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de2ff48e 505e841e-eed2-401d-9583-fef592c977c4 /tidb/cdc/default/default/upstream/7374364445975694966 {"id":7374364445975694966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e841e-eed2-401d-9583-fef592c977c4 {"id":"505e841e-eed2-401d-9583-fef592c977c4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de2ff48e 505e841e-eed2-401d-9583-fef592c977c4 /tidb/cdc/default/default/upstream/7374364445975694966 {"id":7374364445975694966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: old-value-cf Info: {"upstream_id":7374364445975694966,"namespace":"default","id":"old-value-cf","sink_uri":"blackhole:","create_time":"2024-05-29T18:21:21.659905447+08:00","start_ts":450095502211153923,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095502211153923,"checkpoint_ts":450095502211153923,"checkpoint_time":"2024-05-29 18:21:21.555"} Create changefeed successfully! ID: no-old-value-cf Info: {"upstream_id":7374364445975694966,"namespace":"default","id":"no-old-value-cf","sink_uri":"blackhole:","create_time":"2024-05-29T18:21:21.858950032+08:00","start_ts":450095502263582721,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095502263582721,"checkpoint_ts":450095502263582721,"checkpoint_time":"2024-05-29 18:21:21.755"} Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:21:23.190 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:23.199 +08:00] [INFO] [case.go:496] ["all tables synced"] [endTs="2024-05-29 18:21:22"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7831740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42669, start at 2024-05-29 18:21:00.427272662 +0800 CST m=+4.330040415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:25.923 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/29 18:21:25.923 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark1000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:21:27.923 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:28.031 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark1000] [2024/05/29 18:21:28.031 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:27"] [2024/05/29 18:21:28.121 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:28.126 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:28.130 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:28.135 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79cef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42450, start at 2024-05-29 18:21:26.873954635 +0800 CST m=+4.378105729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log [2024/05/29 18:21:28.839 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/29 18:21:28.839 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark2000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f0dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34826, start at 2024-05-29 18:20:39.887485649 +0800 CST m=+4.137857444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:39.893 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:39.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:39.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f250001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:34916, start at 2024-05-29 18:20:40.002195674 +0800 CST m=+4.163935634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:21:29 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.3899138993.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f786aa00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:37030, start at 2024-05-29 18:21:04.054661065 +0800 CST m=+4.132344461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:04.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:04.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f778000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42307, start at 2024-05-29 18:20:40.297434147 +0800 CST m=+4.291443679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.304 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.286 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.286 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f76f8240004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:42396, start at 2024-05-29 18:20:40.332492413 +0800 CST m=+4.278516844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:22:40.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:20:40.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:10:40.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:21:29 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.4645246454.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:30.839 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:30.848 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark2000] [2024/05/29 18:21:30.848 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:30"] [2024/05/29 18:21:30.850 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:30.852 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:30.856 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:30.859 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting CDC state checker... [2024/05/29 18:21:31.224 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/29 18:21:31.224 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark3000] ***************** properties ***************** "workload"="core" "threadcount"="4" "updateproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "dotransactions"="false" "readproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "readallfields"="true" "operationcount"="0" "recordcount"="50" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" ********************************************** Run finished, takes 17.703929ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3695.7, Avg(us): 1411, Min(us): 901, Max(us): 4958, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:21:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2a6f77-456c-4c10-bfec-ed4e92d62a50 {"id":"1c2a6f77-456c-4c10-bfec-ed4e92d62a50","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de5e1d8e 1c2a6f77-456c-4c10-bfec-ed4e92d62a50 /tidb/cdc/default/default/upstream/7374364497413918875 {"id":7374364497413918875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2a6f77-456c-4c10-bfec-ed4e92d62a50 {"id":"1c2a6f77-456c-4c10-bfec-ed4e92d62a50","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de5e1d8e 1c2a6f77-456c-4c10-bfec-ed4e92d62a50 /tidb/cdc/default/default/upstream/7374364497413918875 {"id":7374364497413918875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2a6f77-456c-4c10-bfec-ed4e92d62a50 {"id":"1c2a6f77-456c-4c10-bfec-ed4e92d62a50","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de5e1d8e 1c2a6f77-456c-4c10-bfec-ed4e92d62a50 /tidb/cdc/default/default/upstream/7374364497413918875 {"id":7374364497413918875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: a9b485fd-8c63-4997-a811-7e312611d684 Info: {"upstream_id":7374364497413918875,"namespace":"default","id":"a9b485fd-8c63-4997-a811-7e312611d684","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-29T18:21:32.859025871+08:00","start_ts":450095504325869569,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095504325869569,"checkpoint_ts":450095504325869569,"checkpoint_time":"2024-05-29 18:21:29.622"} [2024/05/29 18:21:33.225 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:21:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79673d13-c7a8-4141-977a-5b915ff346d8 {"id":"79673d13-c7a8-4141-977a-5b915ff346d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de55ad90 79673d13-c7a8-4141-977a-5b915ff346d8 /tidb/cdc/default/default/upstream/7374364489592222571 {"id":7374364489592222571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79673d13-c7a8-4141-977a-5b915ff346d8 {"id":"79673d13-c7a8-4141-977a-5b915ff346d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de55ad90 79673d13-c7a8-4141-977a-5b915ff346d8 /tidb/cdc/default/default/upstream/7374364489592222571 {"id":7374364489592222571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79673d13-c7a8-4141-977a-5b915ff346d8 {"id":"79673d13-c7a8-4141-977a-5b915ff346d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de55ad90 79673d13-c7a8-4141-977a-5b915ff346d8 /tidb/cdc/default/default/upstream/7374364489592222571 {"id":7374364489592222571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:21:34.025 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/29 18:21:34.025 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark4000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (43808) - No such process wait process 43808 exit for 1-th time... process 43808 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Wed May 29 18:21:35 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.4414644148.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/29 18:21:35.234 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:35.245 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark3000] [2024/05/29 18:21:35.245 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:33"] [2024/05/29 18:21:35.246 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:35.332 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:35.336 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:35.345 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:36.025 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:21:36.848 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/29 18:21:36.848 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark5000] Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:21:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {ClusterID:default ID:default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 Info:{"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} Status:0xc000f632d8 TaskPositions:map[25119cdd-98bd-4cec-b38b-b7bd0c024f0f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095505339580420, checkpointTs: 450095505339580420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25119cdd-98bd-4cec-b38b-b7bd0c024f0f {"id":"25119cdd-98bd-4cec-b38b-b7bd0c024f0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fcac 25119cdd-98bd-4cec-b38b-b7bd0c024f0f /tidb/cdc/default/default/changefeed/info/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} /tidb/cdc/default/default/changefeed/status/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":450095505339580420,"min-table-barrier-ts":450095506387894277,"admin-job-type":0} /tidb/cdc/default/default/task/position/25119cdd-98bd-4cec-b38b-b7bd0c024f0f/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {ClusterID:default ID:default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 Info:{"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} Status:0xc000f632d8 TaskPositions:map[25119cdd-98bd-4cec-b38b-b7bd0c024f0f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095505339580420, checkpointTs: 450095505339580420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25119cdd-98bd-4cec-b38b-b7bd0c024f0f {"id":"25119cdd-98bd-4cec-b38b-b7bd0c024f0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fcac 25119cdd-98bd-4cec-b38b-b7bd0c024f0f /tidb/cdc/default/default/changefeed/info/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,E+ grep -q 'failed to get info:' RROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} /tidb/cdc/default/default/changefeed/status/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":450095505339580420,"min-table-barrier-ts":450095506387894277,"admin-job-type":0} /tidb/cdc/default/default/task/position/25119cdd-98bd-4cec-b38b-b7bd0c024f0f/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {ClusterID:default ID:default/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 Info:{"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} Status:0xc000f632d8 TaskPositions:map[25119cdd-98bd-4cec-b38b-b7bd0c024f0f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095505339580420, checkpointTs: 450095505339580420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25119cdd-98bd-4cec-b38b-b7bd0c024f0f {"id":"25119cdd-98bd-4cec-b38b-b7bd0c024f0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dd74fcac 25119cdd-98bd-4cec-b38b-b7bd0c024f0f /tidb/cdc/default/default/changefeed/info/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"upstream-id":7374364246326417892,"namespace":"default","changefeed-id":"8a6b49df-e616-418f-bd7c-b7b1d3ac6658","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:20:35.642421696+08:00","start-ts":450095490161180673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095490174287874} /tidb/cdc/default/default/changefeed/status/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":450095505339580420,"min-table-barrier-ts":450095506387894277,"admin-job-type":0} /tidb/cdc/default/default/task/position/25119cdd-98bd-4cec-b38b-b7bd0c024f0f/8a6b49df-e616-418f-bd7c-b7b1d3ac6658 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374364246326417892 {"id":7374364246326417892,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id [2024/05/29 18:21:38.037 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:38.049 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark4000] [2024/05/29 18:21:38.049 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:36"] [2024/05/29 18:21:38.051 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:38.122 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:38.127 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:38.131 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42645, start at 2024-05-29 18:21:36.604324552 +0800 CST m=+4.285027127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.613 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log "id": "25119cdd-98bd-4cec-b38b-b7bd0c024f0f", "cluster-id": "default" run task successfully capture_id: 25119cdd-98bd-4cec-b38b-b7bd0c024f0f check_processor_table_count http://127.0.0.1:2379 8a6b49df-e616-418f-bd7c-b7b1d3ac6658 25119cdd-98bd-4cec-b38b-b7bd0c024f0f 1 [2024/05/29 18:21:38.848 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] run task successfully Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check_processor_table_count http://127.0.0.1:2379 8a6b49df-e616-418f-bd7c-b7b1d3ac6658 25119cdd-98bd-4cec-b38b-b7bd0c024f0f 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later [2024/05/29 18:21:39.733 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/29 18:21:39.733 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark6000] [2024/05/29 18:21:40.862 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:21:40.932 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark5000] [2024/05/29 18:21:40.932 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:39"] [2024/05/29 18:21:40.935 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:40.937 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:40.941 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:40.948 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check_processor_table_count http://127.0.0.1:2379 8a6b49df-e616-418f-bd7c-b7b1d3ac6658 25119cdd-98bd-4cec-b38b-b7bd0c024f0f 0 [2024/05/29 18:21:41.733 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] run task successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:21:42.527 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/29 18:21:42.527 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark7000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f783184001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42568, start at 2024-05-29 18:21:00.428415524 +0800 CST m=+4.409670835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.435 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7831740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:42669, start at 2024-05-29 18:21:00.427272662 +0800 CST m=+4.330040415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:00.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:00.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:00.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed test leveldb sorter + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.46824.out cli tso query --pd=http://127.0.0.1:2379 ***************** properties ***************** "workload"="core" "dotransactions"="false" "insertproportion"="0" "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "recordcount"="50" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readproportion"="0" "threadcount"="4" "mysql.port"="4000" "scanproportion"="0" ********************************************** Run finished, takes 17.492478ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3693.6, Avg(us): 1392, Min(us): 883, Max(us): 4572, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later + set +x + tso='450095507957088257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095507957088257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/29 18:21:43.746 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:43.758 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark6000] [2024/05/29 18:21:43.758 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:42"] [2024/05/29 18:21:43.822 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:43.825 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:43.828 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:43.832 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:44.527 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later [2024/05/29 18:21:45.626 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/29 18:21:45.626 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark8000] [2024/05/29 18:21:46.539 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:46.550 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark7000] [2024/05/29 18:21:46.550 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:45"] [2024/05/29 18:21:46.554 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:46.625 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:46.629 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:46.633 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:47.626 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:21:47 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:48.524 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/29 18:21:48.524 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark9000] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:21:49.639 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:49.649 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark8000] [2024/05/29 18:21:49.649 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:47"] [2024/05/29 18:21:49.652 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:49.654 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:49.724 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:49.728 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:50.524 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7869fc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:36944, start at 2024-05-29 18:21:04.034344288 +0800 CST m=+4.196401595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:03.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:03.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f786aa00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:37030, start at 2024-05-29 18:21:04.054661065 +0800 CST m=+4.132344461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:04.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:04.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:04.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.41159.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79cef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42450, start at 2024-05-29 18:21:26.873954635 +0800 CST m=+4.378105729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/29 18:21:51.424 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/29 18:21:51.424 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark10000] + set +x + tso='450095510002335745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095510002335745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:21:52.534 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:52.549 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark9000] [2024/05/29 18:21:52.549 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:50"] [2024/05/29 18:21:52.551 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:52.553 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:52.559 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:52.625 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:53.424 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:21:54.324 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/29 18:21:54.324 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark11000] [2024/05/29 18:21:55.433 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:55.443 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark10000] [2024/05/29 18:21:55.443 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:53"] [2024/05/29 18:21:55.445 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:55.447 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:55.523 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:55.527 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:56.324 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:57.079 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/29 18:21:57.079 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark12000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:21:58.334 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:21:58.352 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark11000] [2024/05/29 18:21:58.352 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:56"] [2024/05/29 18:21:58.422 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:58.425 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:21:58.429 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:21:58.433 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore *************************** 1. row *************************** count(distinct region_id): 1 split_and_random_merge scale: 80 [2024/05/29 18:21:59.121 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/29 18:22:00.025 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/29 18:22:00.026 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark13000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42645, start at 2024-05-29 18:21:36.604324552 +0800 CST m=+4.285027127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.613 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/29 18:22:01.132 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:01.155 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark12000] [2024/05/29 18:22:01.155 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:21:59"] [2024/05/29 18:22:01.158 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:01.161 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:01.225 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:01.229 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:22:02.026 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:02.924 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark14000] [2024/05/29 18:22:02.924 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark14000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:22:04.037 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:04.047 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark13000] [2024/05/29 18:22:04.047 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:02"] [2024/05/29 18:22:04.050 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:04.053 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:04.135 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:04.139 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] *************************** 1. row *************************** count(distinct region_id): 59 [2024/05/29 18:22:04.924 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Verifying downstream PD is started... [2024/05/29 18:22:05.772 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/29 18:22:05.772 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark15000] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:22:06.936 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:06.946 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark14000] [2024/05/29 18:22:06.947 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:05"] [2024/05/29 18:22:06.949 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:06.952 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:07.023 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:07.027 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:22:07.821 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:08.538 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark16000] [2024/05/29 18:22:08.538 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark16000] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table region_merge.t1 exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:22:09.831 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:09.841 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark15000] [2024/05/29 18:22:09.841 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:08"] [2024/05/29 18:22:09.843 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:09.846 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:09.850 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:09.854 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:10.538 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79ccac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42340, start at 2024-05-29 18:21:26.719993757 +0800 CST m=+4.285160393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f79cef80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:42450, start at 2024-05-29 18:21:26.873954635 +0800 CST m=+4.378105729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:26.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:26.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:26.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:22:10 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.4662846630.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:22:11 CST 2024] <<<<<< run test case region_merge success! >>>>>> [2024/05/29 18:22:11.625 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/29 18:22:11.625 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark17000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:22:12.549 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:12.625 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark16000] [2024/05/29 18:22:12.626 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:10"] [2024/05/29 18:22:12.628 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:12.631 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:12.636 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:12.641 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:13.625 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:22:13 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07384917-4c1a-4da0-898e-ca11fdca463f {"id":"07384917-4c1a-4da0-898e-ca11fdca463f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3def81090 07384917-4c1a-4da0-898e-ca11fdca463f /tidb/cdc/default/default/upstream/7374364698529912460 {"id":7374364698529912460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07384917-4c1a-4da0-898e-ca11fdca463f {"id":"07384917-4c1a-4da0-898e-ca11fdca463f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3def81090 07384917-4c1a-4da0-898e-ca11fdca463f /tidb/cdc/default/default/upstream/7374364698529912460 {"id":7374364698529912460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07384917-4c1a-4da0-898e-ca11fdca463f {"id":"07384917-4c1a-4da0-898e-ca11fdca463f","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3def81090 07384917-4c1a-4da0-898e-ca11fdca463f /tidb/cdc/default/default/upstream/7374364698529912460 {"id":7374364698529912460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/29 18:22:13.821 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-normal Info: {"upstream_id":7374364698529912460,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-8893?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-29T18:22:13.870354165+08:00","start_ts":450095515076657153,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095515076657153,"checkpoint_ts":450095515076657153,"checkpoint_time":"2024-05-29 18:22:10.633"} [2024/05/29 18:22:13.968 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7374364698529912460,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25701?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-29T18:22:14.020804017+08:00","start_ts":450095515076657153,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095515076657153,"checkpoint_ts":450095515076657153,"checkpoint_time":"2024-05-29 18:22:10.633"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8ed80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45528, start at 2024-05-29 18:22:11.927646143 +0800 CST m=+4.171895689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log [2024/05/29 18:22:14.102 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7374364698529912460,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-20828?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-29T18:22:14.220176589+08:00","start_ts":450095515076657153,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095515076657153,"checkpoint_ts":450095515076657153,"checkpoint_time":"2024-05-29 18:22:10.633"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-8893?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-8893?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25701?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25701?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-20828?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Wed May 29 18:22:14 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-20828?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml [2024/05/29 18:22:14.333 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark18000] [2024/05/29 18:22:14.333 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark18000] [2024/05/29 18:22:15.635 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:15.647 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark17000] [2024/05/29 18:22:15.647 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:13"] [2024/05/29 18:22:15.724 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:15.727 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:15.732 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:15.735 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:22:16.333 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:17.147 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark19000] [2024/05/29 18:22:17.147 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark19000] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed can't found update row with old value run task failed 1-th time, retry later [2024/05/29 18:22:18.342 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:18.353 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark18000] [2024/05/29 18:22:18.353 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:16"] check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed [2024/05/29 18:22:18.448 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516602597378,\"commit-ts\":450095516602597380,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/05/29 18:22:18.449 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516602597391,\"commit-ts\":450095516602597392,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/05/29 18:22:18.449 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516615704578,\"commit-ts\":450095516615704579,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] [2024/05/29 18:22:18.485 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516602597378,\"commit-ts\":450095516602597380,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/05/29 18:22:18.485 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516602597391,\"commit-ts\":450095516602597392,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/05/29 18:22:18.486 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450095516615704578,\"commit-ts\":450095516615704579,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] run task successfully [2024/05/29 18:22:18.421 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:18.425 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:18.429 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:18.436 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:22:19 CST 2024] <<<<<< run test case multi_changefeed success! >>>>>> [2024/05/29 18:22:19.147 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:19.458 +08:00] [INFO] [case.go:554] ["one upstream workload finished"] [round=20001] [2024/05/29 18:22:19.458 +08:00] [INFO] [case.go:600] ["one downstream exit due to receive done"] [2024/05/29 18:22:19.459 +08:00] [INFO] [case.go:554] ["one upstream workload finished"] [round=20002] [2024/05/29 18:22:19.459 +08:00] [INFO] [case.go:600] ["one downstream exit due to receive done"] [2024/05/29 18:22:19.461 +08:00] [INFO] [case.go:554] ["one upstream workload finished"] [round=20003] [2024/05/29 18:22:19.461 +08:00] [INFO] [case.go:600] ["one downstream exit due to receive done"] [2024/05/29 18:22:19.462 +08:00] [INFO] [case.go:554] ["one upstream workload finished"] [round=20004] [2024/05/29 18:22:19.462 +08:00] [INFO] [case.go:600] ["one downstream exit due to receive done"] [2024/05/29 18:22:19.966 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/29 18:22:19.966 +08:00] [INFO] [case.go:554] ["one upstream workload finished"] [round=20004] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/29 18:22:21.158 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:21.171 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark19000] [2024/05/29 18:22:21.171 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:19"] [2024/05/29 18:22:21.174 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:21.177 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:21.183 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:21.186 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:21.186 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark20000] Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:22:23.186 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/29 18:22:23.196 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark20000] [2024/05/29 18:22:23.196 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-29 18:22:21"] [2024/05/29 18:22:23.199 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:23.201 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:23.205 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/29 18:22:23.208 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/29 18:22:23.208 +08:00] [INFO] [case.go:600] ["one downstream exit due to receive done"] [2024/05/29 18:22:23.208 +08:00] [INFO] [case.go:612] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit ++ date + echo '[Wed May 29 18:22:23 CST 2024] <<<<<< run test case bank success! >>>>>>' [Wed May 29 18:22:23 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42561, start at 2024-05-29 18:21:36.604940455 +0800 CST m=+4.369274059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7a67180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:42645, start at 2024-05-29 18:21:36.604324552 +0800 CST m=+4.285027127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:23:36.613 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:21:36.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:11:36.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8ed80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45528, start at 2024-05-29 18:22:11.927646143 +0800 CST m=+4.171895689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "operationcount"="0" "workload"="core" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "requestdistribution"="uniform" "recordcount"="3000" "mysql.user"="root" "dotransactions"="false" "readallfields"="true" "readproportion"="0" "mysql.db"="sorter" "updateproportion"="0" "scanproportion"="0" "threadcount"="10" ********************************************** Run finished, takes 572.372313ms INSERT - Takes(s): 0.6, Count: 3000, OPS: 5289.4, Avg(us): 1847, Min(us): 787, Max(us): 40265, 95th(us): 2000, 99th(us): 39000 [Wed May 29 18:22:40 CST 2024] <<<<<< START cdc server in sorter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.4697446976.out server --log-file /tmp/tidb_cdc_test/sorter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sorter/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t6 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:22:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/183757f3-3519-49ee-9b50-dbe0adb5afd1 {"id":"183757f3-3519-49ee-9b50-dbe0adb5afd1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de8bd313 183757f3-3519-49ee-9b50-dbe0adb5afd1 /tidb/cdc/default/default/upstream/7374364582627173725 {"id":7374364582627173725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/183757f3-3519-49ee-9b50-dbe0adb5afd1 {"id":"183757f3-3519-49ee-9b50-dbe0adb5afd1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de8bd313 183757f3-3519-49ee-9b50-dbe0adb5afd1 /tidb/cdc/default/default/upstream/7374364582627173725 {"id":7374364582627173725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/183757f3-3519-49ee-9b50-dbe0adb5afd1 {"id":"183757f3-3519-49ee-9b50-dbe0adb5afd1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3de8bd313 183757f3-3519-49ee-9b50-dbe0adb5afd1 /tidb/cdc/default/default/upstream/7374364582627173725 {"id":7374364582627173725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.47036.out cli changefeed create -c ticdc-leveldb-sorter-test-29965 --start-ts=450095507957088257 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' --sort-engine=unified Create changefeed successfully! ID: ticdc-leveldb-sorter-test-29965 Info: {"upstream_id":7374364582627173725,"namespace":"default","id":"ticdc-leveldb-sorter-test-29965","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:22:43.498588631+08:00","start_ts":450095507957088257,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095507957088257,"checkpoint_ts":450095507957088257,"checkpoint_time":"2024-05-29 18:21:43.474"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table sorter.usertable not exists for 1-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e7b4c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34939, start at 2024-05-29 18:22:43.421356241 +0800 CST m=+4.202141489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.426 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "450095522741223428" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/29 18:22:41.498 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:22:41.498 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:22:41.498 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095522741223428] [resolvedTs=450095522741223428] [2024/05/29 18:22:41.499 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=223.289ยตs] [2024/05/29 18:22:41.499 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=256.625ยตs] [2024/05/29 18:22:41.499 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:22:41.499 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:22:41.503 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:22:41.505 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:22:41.506 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:22:41.506 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:22:41.506 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:22:41.510 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:22:41.512 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:22:41.512 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=450095522741223428] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:22:41.512 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Wed May 29 18:22:41 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 table sorter.usertable exists table sorter.check1 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e963c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48104, start at 2024-05-29 18:22:45.171690887 +0800 CST m=+4.221972770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.135 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.135 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log Starting CDC state checker... run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later table sorter.check1 not exists for 2-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 table sorter.check1 exists check diff successfully Verifying downstream PD is started... check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 run task successfully Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:22:51 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D3EE8D11EFEA94 < X-Xss-Protection: 1; mode=block < Date: Wed, 29 May 2024 10:22:52 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 27223 0 --:--:-- --:--:-- --:--:-- 27807 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:20:30.741","puller_resolved_ts":"2024-05-29 18:20:31.742","last_synced_ts":"2024-05-29 18:20:26.841","now_ts":"2024-05-29 18:22:42.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:20:30.741","puller_resolved_ts":"2024-05-29' '18:20:31.742","last_synced_ts":"2024-05-29' '18:20:26.841","now_ts":"2024-05-29' '18:22:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:20:30.741","puller_resolved_ts":"2024-05-29' '18:20:31.742","last_synced_ts":"2024-05-29' '18:20:26.841","now_ts":"2024-05-29' '18:22:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully table sorter.check2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8d580024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45439, start at 2024-05-29 18:22:11.840857993 +0800 CST m=+4.143389093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7c8ed80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:45528, start at 2024-05-29 18:22:11.927646143 +0800 CST m=+4.171895689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:11.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:11.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:11.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:22:55 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.4951949521.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table sorter.check2 exists check diff successfully ***************** properties ***************** "readproportion"="0" "recordcount"="3000" "mysql.db"="sorter" "readallfields"="true" "threadcount"="10" "scanproportion"="0" "insertproportion"="0" "mysql.user"="root" "operationcount"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.port"="4000" "workload"="core" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 1.034776294s INSERT - Takes(s): 1.0, Count: 2998, OPS: 2918.5, Avg(us): 3390, Min(us): 877, Max(us): 83836, 95th(us): 3000, 99th(us): 79000 table sorter.check3 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:22:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a37ef0d8-fa35-447e-b366-43f27ea4a69c {"id":"a37ef0d8-fa35-447e-b366-43f27ea4a69c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e290 a37ef0d8-fa35-447e-b366-43f27ea4a69c /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a37ef0d8-fa35-447e-b366-43f27ea4a69c {"id":"a37ef0d8-fa35-447e-b366-43f27ea4a69c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e290 a37ef0d8-fa35-447e-b366-43f27ea4a69c /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a37ef0d8-fa35-447e-b366-43f27ea4a69c {"id":"a37ef0d8-fa35-447e-b366-43f27ea4a69c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e290 a37ef0d8-fa35-447e-b366-43f27ea4a69c /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:22:49 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.4128141283.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:22:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bb88744-8144-4742-84fb-7758387a038d {"id":"2bb88744-8144-4742-84fb-7758387a038d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3deae3819 2bb88744-8144-4742-84fb-7758387a038d /tidb/cdc/default/default/upstream/7374364600647133016 {"id":7374364600647133016,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bb88744-8144-4742-84fb-7758387a038d {"id":"2bb88744-8144-4742-84fb-7758387a038d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3deae3819 2bb88744-8144-4742-84fb-7758387a038d /tidb/cdc/default/default/upstream/7374364600647133016 {"id":7374364600647133016,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2bb88744-8144-4742-84fb-7758387a038d {"id":"2bb88744-8144-4742-84fb-7758387a038d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3deae3819 2bb88744-8144-4742-84fb-7758387a038d /tidb/cdc/default/default/upstream/7374364600647133016 {"id":7374364600647133016,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.41343.out cli changefeed create --start-ts=450095510002335745 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: d1ad72be-aa3e-4b22-9b47-6a190b4ae0c8 Info: {"upstream_id":7374364600647133016,"namespace":"default","id":"d1ad72be-aa3e-4b22-9b47-6a190b4ae0c8","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:22:52.641190353+08:00","start_ts":450095510002335745,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095510002335745,"checkpoint_ts":450095510002335745,"checkpoint_time":"2024-05-29 18:21:51.276"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sorter.check3 exists check diff successfully table sorter.usertable2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sorter.usertable2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa1440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31792, start at 2024-05-29 18:23:02.248504098 +0800 CST m=+4.234287979 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log table sorter.usertable2 exists table sorter.check4 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Starting CDC state checker... Edition: Community Verifying downstream PD is started... Starting Upstream TiKV... Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 TiKV Release Version: 6.5.9 UTC Build Time: 2024-05-10 07:21:43 Edition: Community Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) table sorter.check4 exists check diff successfully start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:23:07 CST 2024] <<<<<< run test case sorter success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd66c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41631, start at 2024-05-29 18:23:05.663326895 +0800 CST m=+4.172823580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 exists table processor_delay.t14 exists table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e7b4c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34939, start at 2024-05-29 18:22:43.421356241 +0800 CST m=+4.202141489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.426 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e963c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48104, start at 2024-05-29 18:22:45.171690887 +0800 CST m=+4.221972770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.135 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.135 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8075240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42181, start at 2024-05-29 18:23:15.807417757 +0800 CST m=+6.100944466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:15.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:15.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:15.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa1440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31792, start at 2024-05-29 18:23:02.248504098 +0800 CST m=+4.234287979 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd66c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41631, start at 2024-05-29 18:23:05.663326895 +0800 CST m=+4.172823580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e97240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48010, start at 2024-05-29 18:22:45.225212637 +0800 CST m=+4.342042509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e963c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:48104, start at 2024-05-29 18:22:45.171690887 +0800 CST m=+4.221972770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:45.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:45.135 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:45.135 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later ***************** properties ***************** "operationcount"="0" "readallfields"="true" "recordcount"="20" "mysql.db"="changefeed_auto_stop_1" "mysql.user"="root" "insertproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "readproportion"="0" "mysql.port"="4000" "workload"="core" "requestdistribution"="uniform" "scanproportion"="0" "updateproportion"="0" "threadcount"="4" ********************************************** Run finished, takes 11.559627ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2929.9, Avg(us): 2125, Min(us): 1116, Max(us): 4731, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "updateproportion"="0" "mysql.port"="4000" "mysql.user"="root" "readallfields"="true" "recordcount"="20" "insertproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "scanproportion"="0" "operationcount"="0" "dotransactions"="false" "threadcount"="4" "mysql.db"="changefeed_auto_stop_2" ********************************************** Run finished, takes 12.676544ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2938.3, Avg(us): 2390, Min(us): 1232, Max(us): 5788, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "operationcount"="0" "mysql.db"="changefeed_auto_stop_3" "threadcount"="4" "readallfields"="true" "insertproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "mysql.host"="127.0.0.1" "mysql.port"="4000" "updateproportion"="0" "dotransactions"="false" "workload"="core" "recordcount"="20" "readproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 12.965518ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2702.7, Avg(us): 2451, Min(us): 1294, Max(us): 5441, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "requestdistribution"="uniform" "workload"="core" "readallfields"="true" "scanproportion"="0" "updateproportion"="0" "recordcount"="20" "insertproportion"="0" "threadcount"="4" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "mysql.db"="changefeed_auto_stop_4" "operationcount"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.136181ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3538.5, Avg(us): 1920, Min(us): 959, Max(us): 4484, 95th(us): 5000, 99th(us): 5000 [Wed May 29 18:23:23 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.4678846790.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:23:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:23:26 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.4684446846.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:23:29 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f445dec-5ef2-42fc-9423-8e74e71d47d6 {"id":"9f445dec-5ef2-42fc-9423-8e74e71d47d6","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f73 9f445dec-5ef2-42fc-9423-8e74e71d47d6 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f445dec-5ef2-42fc-9423-8e74e71d47d6 {"id":"9f445dec-5ef2-42fc-9423-8e74e71d47d6","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f73 9f445dec-5ef2-42fc-9423-8e74e71d47d6 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f445dec-5ef2-42fc-9423-8e74e71d47d6 {"id":"9f445dec-5ef2-42fc-9423-8e74e71d47d6","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/c3373e6a-f311-4ada-b935-33a978123f83 {"id":"c3373e6a-f311-4ada-b935-33a978123f83","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f51 c3373e6a-f311-4ada-b935-33a978123f83 /tidb/cdc/default/__cdc_meta__/owner/22318fc3df2a9f73 9f445dec-5ef2-42fc-9423-8e74e71d47d6 /tidb/cdc/default/default/upstream/7374364743802966556 {"id":7374364743802966556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 f524b3e6-13a4-4c0e-9718-b1e3789082ff normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=f524b3e6-13a4-4c0e-9718-b1e3789082ff + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f524b3e6-13a4-4c0e-9718-b1e3789082ff -s + info='{ "upstream_id": 7374364743802966556, "namespace": "default", "id": "f524b3e6-13a4-4c0e-9718-b1e3789082ff", "state": "normal", "checkpoint_tso": 450095518704730113, "checkpoint_time": "2024-05-29 18:22:24.473", "error": null }' + echo '{ "upstream_id": 7374364743802966556, "namespace": "default", "id": "f524b3e6-13a4-4c0e-9718-b1e3789082ff", "state": "normal", "checkpoint_tso": 450095518704730113, "checkpoint_time": "2024-05-29 18:22:24.473", "error": null }' { "upstream_id": 7374364743802966556, "namespace": "default", "id": "f524b3e6-13a4-4c0e-9718-b1e3789082ff", "state": "normal", "checkpoint_tso": 450095518704730113, "checkpoint_time": "2024-05-29 18:22:24.473", "error": null } ++ echo '{' '"upstream_id":' 7374364743802966556, '"namespace":' '"default",' '"id":' '"f524b3e6-13a4-4c0e-9718-b1e3789082ff",' '"state":' '"normal",' '"checkpoint_tso":' 450095518704730113, '"checkpoint_time":' '"2024-05-29' '18:22:24.473",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374364743802966556, '"namespace":' '"default",' '"id":' '"f524b3e6-13a4-4c0e-9718-b1e3789082ff",' '"state":' '"normal",' '"checkpoint_tso":' 450095518704730113, '"checkpoint_time":' '"2024-05-29' '18:22:24.473",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e79cc001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34852, start at 2024-05-29 18:22:43.343878608 +0800 CST m=+4.196693769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7e7b4c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:34939, start at 2024-05-29 18:22:43.421356241 +0800 CST m=+4.202141489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:24:43.426 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:22:43.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:12:43.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.39014.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095536569843713 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095536569843713 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:23:32 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.3905939061.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f818ddc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48437, start at 2024-05-29 18:23:33.793925082 +0800 CST m=+7.023402907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.800 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 not exists for 1-th check, retry later table multi_tables_ddl_test.t66 not exists for 2-th check, retry later table multi_tables_ddl_test.t66 not exists for 3-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:23:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aff42ed9-985e-48a5-866f-31843a92a4c2 {"id":"aff42ed9-985e-48a5-866f-31843a92a4c2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e038258d aff42ed9-985e-48a5-866f-31843a92a4c2 /tidb/cdc/default/default/upstream/7374365020840166746 {"id":7374365020840166746,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aff42ed9-985e-48a5-866f-31843a92a4c2 {"id":"aff42ed9-985e-48a5-866f-31843a92a4c2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e038258d aff42ed9-985e-48a5-866f-31843a92a4c2 /tidb/cdc/default/default/upstream/7374365020840166746 {"id":7374365020840166746,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aff42ed9-985e-48a5-866f-31843a92a4c2 {"id":"aff42ed9-985e-48a5-866f-31843a92a4c2","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e038258d aff42ed9-985e-48a5-866f-31843a92a4c2 /tidb/cdc/default/default/upstream/7374365020840166746 {"id":7374365020840166746,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.39130.out cli changefeed create --start-ts=450095536569843713 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: b6885349-15f8-42d4-b0d2-473097f01266 Info: {"upstream_id":7374365020840166746,"namespace":"default","id":"b6885349-15f8-42d4-b0d2-473097f01266","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-29T18:23:36.314630541+08:00","start_ts":450095536569843713,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095536569843713,"checkpoint_ts":450095536569843713,"checkpoint_time":"2024-05-29 18:23:32.623"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading k8s.io/api v0.24.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading github.com/go-logr/logr v1.2.0 Starting CDC state checker... table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8075240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42181, start at 2024-05-29 18:23:15.807417757 +0800 CST m=+6.100944466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:15.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:15.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:15.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:23:42 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table multi_tables_ddl_test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table multi_tables_ddl_test.finish_mark not exists for 5-th check, retry later table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa2300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31704, start at 2024-05-29 18:23:02.287525606 +0800 CST m=+4.319227870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fa1440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:31792, start at 2024-05-29 18:23:02.248504098 +0800 CST m=+4.234287979 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:02.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:02.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:02.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.35770.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095539970113537 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095539970113537 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095539970113537 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:23:45 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3581135813.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.finish_mark exists check table exists success Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095540006289412, "checkpoint_time": "2024-05-29 18:23:45.732", "error": null }' + echo '{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095540006289412, "checkpoint_time": "2024-05-29 18:23:45.732", "error": null }' { "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450095540006289412, "checkpoint_time": "2024-05-29 18:23:45.732", "error": null } ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450095540006289412, '"checkpoint_time":' '"2024-05-29' '18:23:45.732",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450095540006289412, '"checkpoint_time":' '"2024-05-29' '18:23:45.732",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095540189790209, "checkpoint_time": "2024-05-29 18:23:46.432", "error": null }' + echo '{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095540189790209, "checkpoint_time": "2024-05-29 18:23:46.432", "error": null }' { "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450095540189790209, "checkpoint_time": "2024-05-29 18:23:46.432", "error": null } ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450095540189790209, '"checkpoint_time":' '"2024-05-29' '18:23:46.432",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450095540189790209, '"checkpoint_time":' '"2024-05-29' '18:23:46.432",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095535025815555, "checkpoint_time": "2024-05-29 18:23:26.733", "error": { "time": "2024-05-29T18:23:27.88202106+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095535025815564]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095535025815555, "checkpoint_time": "2024-05-29 18:23:26.733", "error": { "time": "2024-05-29T18:23:27.88202106+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095535025815564]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7374364698529912460, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450095535025815555, "checkpoint_time": "2024-05-29 18:23:26.733", "error": { "time": "2024-05-29T18:23:27.88202106+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095535025815564]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450095535025815555, '"checkpoint_time":' '"2024-05-29' '18:23:26.733",' '"error":' '{' '"time":' '"2024-05-29T18:23:27.88202106+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450095535025815564]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374364698529912460, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450095535025815555, '"checkpoint_time":' '"2024-05-29' '18:23:26.733",' '"error":' '{' '"time":' '"2024-05-29T18:23:27.88202106+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450095535025815564]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095535025815564]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-29 18:23:26.733 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450095535025815564. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450095535025815564]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:23:48 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd5c8001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41538, start at 2024-05-29 18:23:05.624380279 +0800 CST m=+4.199574069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f7fd66c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:41631, start at 2024-05-29 18:23:05.663326895 +0800 CST m=+4.172823580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:05.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:05.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:05.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:23:48 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4564145643.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:23:48 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/206db67a-37c7-4a3a-af3d-9a28a36a107d {"id":"206db67a-37c7-4a3a-af3d-9a28a36a107d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e06acf8e 206db67a-37c7-4a3a-af3d-9a28a36a107d /tidb/cdc/default/default/upstream/7374365107643613180 {"id":7374365107643613180,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/206db67a-37c7-4a3a-af3d-9a28a36a107d {"id":"206db67a-37c7-4a3a-af3d-9a28a36a107d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e06acf8e 206db67a-37c7-4a3a-af3d-9a28a36a107d /tidb/cdc/default/default/upstream/7374365107643613180 {"id":7374365107643613180,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/206db67a-37c7-4a3a-af3d-9a28a36a107d {"id":"206db67a-37c7-4a3a-af3d-9a28a36a107d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e06acf8e 206db67a-37c7-4a3a-af3d-9a28a36a107d /tidb/cdc/default/default/upstream/7374365107643613180 {"id":7374365107643613180,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095539970113537 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.35878.out cli changefeed create --start-ts=450095539970113537 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374365107643613180,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:23:49.314281119+08:00","start_ts":450095539970113537,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095539970113537,"checkpoint_ts":450095539970113537,"checkpoint_time":"2024-05-29 18:23:45.594"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:23:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad841b7-10be-42ad-8c99-c1943e3fa024 {"id":"8ad841b7-10be-42ad-8c99-c1943e3fa024","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c58e 8ad841b7-10be-42ad-8c99-c1943e3fa024 /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad841b7-10be-42ad-8c99-c1943e3fa024 {"id":"8ad841b7-10be-42ad-8c99-c1943e3fa024","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c58e 8ad841b7-10be-42ad-8c99-c1943e3fa024 /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad841b7-10be-42ad-8c99-c1943e3fa024 {"id":"8ad841b7-10be-42ad-8c99-c1943e3fa024","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c58e 8ad841b7-10be-42ad-8c99-c1943e3fa024 /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later table test.table1 not exists for 1-th check, retry later table test.table1 exists table test.table2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table2 exists check diff failed 1-th time, retry later table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f818ddc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48437, start at 2024-05-29 18:23:33.793925082 +0800 CST m=+7.023402907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.800 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later check diff successfully Resume changefeed with checkpointTs3 450095543669227521 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.table5 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... table test.table5 not exists for 2-th check, retry later Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8058540021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42084, start at 2024-05-29 18:23:13.99001454 +0800 CST m=+4.340438479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:13.996 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:13.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:13.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8075240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:42181, start at 2024-05-29 18:23:15.807417757 +0800 CST m=+6.100944466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:15.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:15.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:15.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.46715.out cli tso query --pd=http://127.0.0.1:2379 table test.table5 not exists for 3-th check, retry later + set +x + tso='450095544864866305 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095544864866305 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:24:04 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.4675746759.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358180016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:48045, start at 2024-05-29 18:24:03.10928491 +0800 CST m=+4.359511460 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.116 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/error.log table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table5 exists table test.table6 not exists for 1-th check, retry later Starting CDC state checker... table processor_delay.t50 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:24:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a561102-9d9c-47e8-abbb-e88e8365fde8 {"id":"8a561102-9d9c-47e8-abbb-e88e8365fde8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0b1cb8e 8a561102-9d9c-47e8-abbb-e88e8365fde8 /tidb/cdc/default/default/upstream/7374365157421552969 {"id":7374365157421552969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a561102-9d9c-47e8-abbb-e88e8365fde8 {"id":"8a561102-9d9c-47e8-abbb-e88e8365fde8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0b1cb8e 8a561102-9d9c-47e8-abbb-e88e8365fde8 /tidb/cdc/default/default/upstream/7374365157421552969 {"id":7374365157421552969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8a561102-9d9c-47e8-abbb-e88e8365fde8 {"id":"8a561102-9d9c-47e8-abbb-e88e8365fde8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0b1cb8e 8a561102-9d9c-47e8-abbb-e88e8365fde8 /tidb/cdc/default/default/upstream/7374365157421552969 {"id":7374365157421552969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.46821.out cli changefeed create --start-ts=450095544864866305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: e85b3b4f-7305-4bdb-831b-bab66dd55f2a Info: {"upstream_id":7374365157421552969,"namespace":"default","id":"e85b3b4f-7305-4bdb-831b-bab66dd55f2a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:24:07.936001271+08:00","start_ts":450095544864866305,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095544864866305,"checkpoint_ts":450095544864866305,"checkpoint_time":"2024-05-29 18:24:04.266"} PASS # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.table6 exists check diff successfully Resume changefeed with checkpointTs1 450095527718027270 + set +x table test.table3 not exists for 1-th check, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:24:10 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table3 exists table test.table4 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f9680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48272, start at 2024-05-29 18:24:13.448682901 +0800 CST m=+5.334989378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/error.log table test.table4 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f81904c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48350, start at 2024-05-29 18:23:33.95569049 +0800 CST m=+7.318638816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f818ddc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--764rd, pid:48437, start at 2024-05-29 18:23:33.793925082 +0800 CST m=+7.023402907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:25:33.800 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:23:33.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:13:33.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.52624.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095548630302721 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095548630302721 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:24:19 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.5034350345.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_panic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:24:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/312d1416-fd8f-43b7-961d-463c0a319565 {"id":"312d1416-fd8f-43b7-961d-463c0a319565","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e462 312d1416-fd8f-43b7-961d-463c0a319565 /tidb/cdc/default/default/changefeed/info/f742423d-a64c-42ac-b49b-5872d12f822d {"upstream-id":7374364882058272463,"namespace":"default","changefeed-id":"f742423d-a64c-42ac-b49b-5872d12f822d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:22:58.794999135+08:00","start-ts":450095527718027270,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-29T18:24:20.225725186+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095549043441668} /tidb/cdc/default/default/changefeed/status/f742423d-a64c-42ac-b49b-5872d12f822d {"checkpoint-ts":450095543669227552,"min-table-barrier-ts":450095543708811286,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/312d1416-fd8f-43b7-961d-463c0a319565 {"id":"312d1416-fd8f-43b7-961d-463c0a319565","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e462 312d1416-fd8f-43b7-961d-463c0a319565 /tidb/cdc/default/default/changefeed/info/f742423d-a64c-42ac-b49b-5872d12f822d {"upstream-id":7374364882058272463,"namespace":"default","changefeed-id":"f742423d-a64c-42ac-b49b-5872d12f822d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:22:58.794999135+08:00","start-ts":450095527718027270,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-29T18:24:20.225725186+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095549043441668} /tidb/cdc/default/default/changefeed/status/f742423d-a64c-42ac-b49b-5872d12f822d {"checkpoint-ts":450095543669227552,"min-table-barrier-ts":450095543708811286,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/312d1416-fd8f-43b7-961d-463c0a319565 {"id":"312d1416-fd8f-43b7-961d-463c0a319565","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3dfa9e462 312d1416-fd8f-43b7-961d-463c0a319565 /tidb/cdc/default/default/changefeed/info/f742423d-a64c-42ac-b49b-5872d12f822d {"upstream-id":7374364882058272463,"namespace":"default","changefeed-id":"f742423d-a64c-42ac-b49b-5872d12f822d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:22:58.794999135+08:00","start-ts":450095527718027270,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-29T18:24:20.225725186+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095549043441668} /tidb/cdc/default/default/changefeed/status/f742423d-a64c-42ac-b49b-5872d12f822d {"checkpoint-ts":450095543669227552,"min-table-barrier-ts":450095543708811286,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374364882058272463 {"id":7374364882058272463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 f742423d-a64c-42ac-b49b-5872d12f822d failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=f742423d-a64c-42ac-b49b-5872d12f822d + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f742423d-a64c-42ac-b49b-5872d12f822d -s + info='{ "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "failed", "checkpoint_tso": 450095543669227552, "checkpoint_time": "2024-05-29 18:23:59.705", "error": { "time": "2024-05-29T18:24:20.225725186+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "failed", "checkpoint_tso": 450095543669227552, "checkpoint_time": "2024-05-29 18:23:59.705", "error": { "time": "2024-05-29T18:24:20.225725186+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "failed", "checkpoint_tso": 450095543669227552, "checkpoint_time": "2024-05-29 18:23:59.705", "error": { "time": "2024-05-29T18:24:20.225725186+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374364882058272463, '"namespace":' '"default",' '"id":' '"f742423d-a64c-42ac-b49b-5872d12f822d",' '"state":' '"failed",' '"checkpoint_tso":' 450095543669227552, '"checkpoint_time":' '"2024-05-29' '18:23:59.705",' '"error":' '{' '"time":' '"2024-05-29T18:24:20.225725186+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7374364882058272463, '"namespace":' '"default",' '"id":' '"f742423d-a64c-42ac-b49b-5872d12f822d",' '"state":' '"failed",' '"checkpoint_tso":' 450095543669227552, '"checkpoint_time":' '"2024-05-29' '18:23:59.705",' '"error":' '{' '"time":' '"2024-05-29T18:24:20.225725186+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully check_changefeed_state http://127.0.0.1:2379 f742423d-a64c-42ac-b49b-5872d12f822d normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=f742423d-a64c-42ac-b49b-5872d12f822d + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f742423d-a64c-42ac-b49b-5872d12f822d -s + info='{ "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "normal", "checkpoint_tso": 450095549856088066, "checkpoint_time": "2024-05-29 18:24:23.306", "error": null }' + echo '{ "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "normal", "checkpoint_tso": 450095549856088066, "checkpoint_time": "2024-05-29 18:24:23.306", "error": null }' { "upstream_id": 7374364882058272463, "namespace": "default", "id": "f742423d-a64c-42ac-b49b-5872d12f822d", "state": "normal", "checkpoint_tso": 450095549856088066, "checkpoint_time": "2024-05-29 18:24:23.306", "error": null } ++ echo '{' '"upstream_id":' 7374364882058272463, '"namespace":' '"default",' '"id":' '"f742423d-a64c-42ac-b49b-5872d12f822d",' '"state":' '"normal",' '"checkpoint_tso":' 450095549856088066, '"checkpoint_time":' '"2024-05-29' '18:24:23.306",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374364882058272463, '"namespace":' '"default",' '"id":' '"f742423d-a64c-42ac-b49b-5872d12f822d",' '"state":' '"normal",' '"checkpoint_tso":' 450095549856088066, '"checkpoint_time":' '"2024-05-29' '18:24:23.306",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table test1.table1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test1.table1 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test1.table1 exists table test1.table2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:24:31 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358180016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:48045, start at 2024-05-29 18:24:03.10928491 +0800 CST m=+4.359511460 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.116 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/29 18:24:31.930 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/05/29 18:24:31.932 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/29 18:24:31.932 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7374365020840166746] [2024/05/29 18:24:31.932 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/05/29 18:24:31.932 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/05/29 18:24:31.933 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/29 18:24:31.933 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7374365020840166746] [2024/05/29 18:24:31.934 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/05/29 18:24:31.934 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/29 18:24:31.935 +08:00] [INFO] [region_cache.go:2485] ["change store resolve state"] [store=2] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.52271.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095551873810433 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095551873810433 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:24:31 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.5231152313.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:24:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47d04661-3622-4500-bb12-c505a1e20ab8 {"id":"47d04661-3622-4500-bb12-c505a1e20ab8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0316f43 47d04661-3622-4500-bb12-c505a1e20ab8 /tidb/cdc/default/default/upstream/7374365034241233048 {"id":7374365034241233048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47d04661-3622-4500-bb12-c505a1e20ab8 {"id":"47d04661-3622-4500-bb12-c505a1e20ab8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0316f43 47d04661-3622-4500-bb12-c505a1e20ab8 /tidb/cdc/default/default/upstream/7374365034241233048 {"id":7374365034241233048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47d04661-3622-4500-bb12-c505a1e20ab8 {"id":"47d04661-3622-4500-bb12-c505a1e20ab8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0316f43 47d04661-3622-4500-bb12-c505a1e20ab8 /tidb/cdc/default/default/upstream/7374365034241233048 {"id":7374365034241233048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.52371.out cli changefeed create --start-ts=450095551873810433 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0d42fcc5-0d12-4812-81ea-dfda8416adcf Info: {"upstream_id":7374365034241233048,"namespace":"default","id":"0d42fcc5-0d12-4812-81ea-dfda8416adcf","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:24:34.889573549+08:00","start_ts":450095551873810433,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095551873810433,"checkpoint_ts":450095551873810433,"checkpoint_time":"2024-05-29 18:24:31.003"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table batch_add_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f85713c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48644, start at 2024-05-29 18:24:37.470474791 +0800 CST m=+4.234424914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_panic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_panic/tiflash/log/error.log table batch_add_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:24:40 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:24:41.946 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/29 18:24:41.947 +08:00] [INFO] [base_client.go:143] ["[pd] exit member loop due to context canceled"] [2024/05/29 18:24:41.947 +08:00] [INFO] [client.go:783] ["[pd] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/29 18:24:41.947 +08:00] [INFO] [client.go:719] ["[pd] exit tso dispatcher"] [dc-location=global] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f9680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48272, start at 2024-05-29 18:24:13.448682901 +0800 CST m=+5.334989378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later table test.t2 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 2-th check, retry later table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 35614 1.0 0.0 1983508 121264 ? Sl 18:23 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716978223217228826.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 35618 1.0 0.0 2222120 128132 ? Sl 18:23 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716978223219932907.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 35709 1.0 0.0 1803804 137780 ? Sl 18:23 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716978223290927772.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4672 0 --:--:-- --:--:-- --:--:-- 4764 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:24:49.444","puller_resolved_ts":"2024-05-29 18:24:49.444","last_synced_ts":"2024-05-29 18:24:43.593","now_ts":"2024-05-29 18:24:50.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:24:49.444","puller_resolved_ts":"2024-05-29' '18:24:49.444","last_synced_ts":"2024-05-29' '18:24:43.593","now_ts":"2024-05-29' '18:24:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:24:49.444","puller_resolved_ts":"2024-05-29' '18:24:49.444","last_synced_ts":"2024-05-29' '18:24:43.593","now_ts":"2024-05-29' '18:24:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:47962, start at 2024-05-29 18:24:03.176471236 +0800 CST m=+4.497529220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8358180016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:48045, start at 2024-05-29 18:24:03.10928491 +0800 CST m=+4.359511460 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:03.116 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:03.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:03.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 table test.t2 not exists for 3-th check, retry later table consistent_replicate_gbk.check1 not exists for 2-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.52042.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095557374377985 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095557374377985 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_gbk.check1 exists check diff successfully table test.t2 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:24:54 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4591445916.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868bcc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51827, start at 2024-05-29 18:24:55.550601292 +0800 CST m=+4.305799750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:24:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/da81e3e3-e45d-48ac-8143-5cfde1e35554 {ClusterID:default ID:default/da81e3e3-e45d-48ac-8143-5cfde1e35554 Info:{"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} Status:0xc0013a74e8 TaskPositions:map[2b5c062e-4ac6-4009-a63f-093df19b0f74:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating tableID: 84, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b5c062e-4ac6-4009-a63f-093df19b0f74 {"id":"2b5c062e-4ac6-4009-a63f-093df19b0f74","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c702 2b5c062e-4ac6-4009-a63f-093df19b0f74 /tidb/cdc/default/default/changefeed/info/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} /tidb/cdc/default/default/changefeed/status/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":450095556115300389,"min-table-barrier-ts":450095558592561157,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b5c062e-4ac6-4009-a63f-093df19b0f74/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/da81e3e3-e45d-48ac-8143-5cfde1e35554 {ClusterID:default ID:default/da81e3e3-e45d-48ac-8143-5cfde1e35554 Info:{"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} Status:0xc0013a74e8 TaskPositions:map[2b5c062e-4ac6-4009-a63f-093df19b0f74:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating tableID: 84, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b5c062e-4ac6-4009-a63f-093df19b0f74 {"id":"2b5c062e-4ac6-4009-a63f-093df19b0f74","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c702 2b5c062e-4ac6-4009-a63f-093df19b0f74 /tidb/cdc/default/default/changefeed/info/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} /tidb/cdc/default/default/changefeed/status/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":450095556115300389,"min-table-barrier-ts":450095558592561157,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b5c062e-4ac6-4009-a63f-093df19b0f74/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/da81e3e3-e45d-48ac-8143-5cfde1e35554 {ClusterID:default ID:default/da81e3e3-e45d-48ac-8143-5cfde1e35554 Info:{"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} Status:0xc0013a74e8 TaskPositions:map[2b5c062e-4ac6-4009-a63f-093df19b0f74:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating tableID: 84, resolvedTs: 450095556115300389, checkpointTs: 450095556115300389, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b5c062e-4ac6-4009-a63f-093df19b0f74 {"id":"2b5c062e-4ac6-4009-a63f-093df19b0f74","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e072c702 2b5c062e-4ac6-4009-a63f-093df19b0f74 /tidb/cdc/default/default/changefeed/info/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"upstream-id":7374365124835648512,"namespace":"default","changefeed-id":"da81e3e3-e45d-48ac-8143-5cfde1e35554","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-29T18:23:51.589427507+08:00","start-ts":450095541513879558,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095541540093953} /tidb/cdc/default/default/changefeed/status/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":450095556115300389,"min-table-barrier-ts":450095558592561157,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b5c062e-4ac6-4009-a63f-093df19b0f74/da81e3e3-e45d-48ac-8143-5cfde1e35554 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374365124835648512 {"id":7374365124835648512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table test.t2 not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table test.t2 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.t2 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check_redo_resolved_ts da81e3e3-e45d-48ac-8143-5cfde1e35554 450095560322711558 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/da81e3e3-e45d-48ac-8143-5cfde1e35554/meta global resolved ts 450095560165425157 not forward to 450095560322711558 run task failed 1-th time, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 9-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f8c80001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48173, start at 2024-05-29 18:24:13.366993459 +0800 CST m=+5.316556679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f83f9680019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--n6lsh, pid:48272, start at 2024-05-29 18:24:13.448682901 +0800 CST m=+5.334989378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:13.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:13.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:13.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check_redo_resolved_ts da81e3e3-e45d-48ac-8143-5cfde1e35554 450095560322711558 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/da81e3e3-e45d-48ac-8143-5cfde1e35554/meta run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/29 18:25:06.051 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:25:06.053 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:25:06.056 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:25:06.056 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095558854705167] [resolvedTs=450095560689713158] [2024/05/29 18:25:06.056 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:25:06.056 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:25:06.056 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:25:06.057 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/29 18:25:06.060 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.403894ms] [2024/05/29 18:25:06.061 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:25:06.061 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=5.309406ms] [2024/05/29 18:25:06.063 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:25:06.064 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:25:06.065 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:25:06.065 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:25:06.064 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:25:06.069 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:25:06.071 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:25:06.071 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450095558854705162,\"CommitTs\":450095558854705167,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/29 18:25:06.072 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450095558854705162,\"CommitTs\":450095558854705167,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/29 18:25:06.073 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450095558854705162] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050: Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] table foreign_key.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... table test.t2 not exists for 10-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/29 18:25:06.273 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450095560689713158] [2024/05/29 18:25:06.273 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450095560689713158] [2024/05/29 18:25:06.374 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=450095560689713158] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:25:06.374 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully check diff successfully [Wed May 29 18:25:06 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT [Wed May 29 18:25:06 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5242852430.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.t2 not exists for 11-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53392, start at 2024-05-29 18:25:06.087211703 +0800 CST m=+4.327219458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:25:09 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5248652488.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting CDC state checker... table test.t2 not exists for 12-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f85713c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48644, start at 2024-05-29 18:24:37.470474791 +0800 CST m=+4.234424914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table test.t2 exists check diff successfully table foreign_key.finish_mark not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:25:12 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:12 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:25:12 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5255952561.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:15 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e649d768-437e-4e21-b431-bd1cc07d7fcb {"id":"e649d768-437e-4e21-b431-bd1cc07d7fcb","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8c3 e649d768-437e-4e21-b431-bd1cc07d7fcb /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e649d768-437e-4e21-b431-bd1cc07d7fcb {"id":"e649d768-437e-4e21-b431-bd1cc07d7fcb","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8c3 e649d768-437e-4e21-b431-bd1cc07d7fcb /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b155c048-cdcc-476d-be1c-27ac3ff62165 {"id":"b155c048-cdcc-476d-be1c-27ac3ff62165","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/e649d768-437e-4e21-b431-bd1cc07d7fcb {"id":"e649d768-437e-4e21-b431-bd1cc07d7fcb","address":"127.0.0.1:8302","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f8c4bfd9-dd88-421d-a884-9d887e4884d9 {"id":"f8c4bfd9-dd88-421d-a884-9d887e4884d9","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd88d b155c048-cdcc-476d-be1c-27ac3ff62165 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8b1 f8c4bfd9-dd88-421d-a884-9d887e4884d9 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e19dd8c3 e649d768-437e-4e21-b431-bd1cc07d7fcb /tidb/cdc/default/default/upstream/7374365408662132781 {"id":7374365408662132781,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.cli.52617.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 334e92a3-b39d-4b40-8466-c20e56f423c0 Info: {"upstream_id":7374365408662132781,"namespace":"default","id":"334e92a3-b39d-4b40-8466-c20e56f423c0","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-29T18:25:16.079258241+08:00","start_ts":450095563660853250,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095563660853250,"checkpoint_ts":450095563660853250,"checkpoint_time":"2024-05-29 18:25:15.967"} PASS table foreign_key.finish_mark not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table foreign_key.finish_mark not exists for 8-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table foreign_key.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... table foreign_key.finish_mark not exists for 10-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table foreign_key.finish_mark not exists for 11-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868bcc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51827, start at 2024-05-29 18:24:55.550601292 +0800 CST m=+4.305799750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table foreign_key.finish_mark exists check diff successfully [Wed May 29 18:25:17 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.5274052742.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f61854cc-4bd1-48cf-9a42-e6d6c94a808d {"id":"f61854cc-4bd1-48cf-9a42-e6d6c94a808d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0e78c1f f61854cc-4bd1-48cf-9a42-e6d6c94a808d /tidb/cdc/default/default/upstream/7374365229365537104 {"id":7374365229365537104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f61854cc-4bd1-48cf-9a42-e6d6c94a808d {"id":"f61854cc-4bd1-48cf-9a42-e6d6c94a808d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0e78c1f f61854cc-4bd1-48cf-9a42-e6d6c94a808d /tidb/cdc/default/default/upstream/7374365229365537104 {"id":7374365229365537104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f61854cc-4bd1-48cf-9a42-e6d6c94a808d {"id":"f61854cc-4bd1-48cf-9a42-e6d6c94a808d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e0e78c1f f61854cc-4bd1-48cf-9a42-e6d6c94a808d /tidb/cdc/default/default/upstream/7374365229365537104 {"id":7374365229365537104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.52805.out cli changefeed create --start-ts=450095548630302721 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: d9e52045-d954-404a-b2b7-a120ff50f420 Info: {"upstream_id":7374365229365537104,"namespace":"default","id":"d9e52045-d954-404a-b2b7-a120ff50f420","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:25:20.872172065+08:00","start_ts":450095548630302721,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095548630302721,"checkpoint_ts":450095548630302721,"checkpoint_time":"2024-05-29 18:24:18.630"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table split_region.test1 not exists for 1-th check, retry later table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table split_region.test2 exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:25:26 CST 2024] <<<<<< run test case foreign_key success! >>>>>> Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f856f0c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48554, start at 2024-05-29 18:24:37.350607166 +0800 CST m=+4.190122621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.356 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f85713c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--9tlz5, pid:48644, start at 2024-05-29 18:24:37.470474791 +0800 CST m=+4.234424914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:37.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:37.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:37.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.52584.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095567615033345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095567615033345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:25:31 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ true == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + true + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.5261752619.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a7bc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47157, start at 2024-05-29 18:25:30.111832814 +0800 CST m=+4.632027291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:30.118 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:30.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:30.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53392, start at 2024-05-29 18:25:06.087211703 +0800 CST m=+4.327219458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:25:34 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.526225267852680.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:37 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0185e31d-b4b2-4191-8f37-676f66fe5bab {"id":"0185e31d-b4b2-4191-8f37-676f66fe5bab","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e20437b2 0185e31d-b4b2-4191-8f37-676f66fe5bab /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0185e31d-b4b2-4191-8f37-676f66fe5bab {"id":"0185e31d-b4b2-4191-8f37-676f66fe5bab","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e20437b2 0185e31d-b4b2-4191-8f37-676f66fe5bab /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0185e31d-b4b2-4191-8f37-676f66fe5bab {"id":"0185e31d-b4b2-4191-8f37-676f66fe5bab","address":"127.0.0.1:8301","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/capture/f2c2a8ab-885f-489d-86e2-dda71ddd8876 {"id":"f2c2a8ab-885f-489d-86e2-dda71ddd8876","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e204378f f2c2a8ab-885f-489d-86e2-dda71ddd8876 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e20437b2 0185e31d-b4b2-4191-8f37-676f66fe5bab /tidb/cdc/default/default/upstream/7374365521238145952 {"id":7374365521238145952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.52744.out cli changefeed create --start-ts=450095567615033345 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: d92a37e4-1065-47ba-963b-0aea962dcb2f Info: {"upstream_id":7374365521238145952,"namespace":"default","id":"d92a37e4-1065-47ba-963b-0aea962dcb2f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:25:37.831063392+08:00","start_ts":450095567615033345,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095567615033345,"checkpoint_ts":450095567615033345,"checkpoint_time":"2024-05-29 18:25:31.051"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43093, start at 2024-05-29 18:25:37.533151793 +0800 CST m=+4.302525733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.540 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:25:40 CST 2024] <<<<<< run test case split_region success! >>>>>> Starting CDC state checker... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868b6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51726, start at 2024-05-29 18:24:55.524576357 +0800 CST m=+4.362595468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f868bcc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:51827, start at 2024-05-29 18:24:55.550601292 +0800 CST m=+4.305799750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:26:55.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:24:55.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:14:55.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:25:45 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.5584955851.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:48 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b65624bb-39c7-42d0-a1fa-b266519e4db4 {"id":"b65624bb-39c7-42d0-a1fa-b266519e4db4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e23c778d b65624bb-39c7-42d0-a1fa-b266519e4db4 /tidb/cdc/default/default/upstream/7374365585732907286 {"id":7374365585732907286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b65624bb-39c7-42d0-a1fa-b266519e4db4 {"id":"b65624bb-39c7-42d0-a1fa-b266519e4db4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e23c778d b65624bb-39c7-42d0-a1fa-b266519e4db4 /tidb/cdc/default/default/upstream/7374365585732907286 {"id":7374365585732907286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b65624bb-39c7-42d0-a1fa-b266519e4db4 {"id":"b65624bb-39c7-42d0-a1fa-b266519e4db4","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e23c778d b65624bb-39c7-42d0-a1fa-b266519e4db4 /tidb/cdc/default/default/upstream/7374365585732907286 {"id":7374365585732907286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 16f93d2f-819d-4245-a184-ac921bcb0fc0 Info: {"upstream_id":7374365585732907286,"namespace":"default","id":"16f93d2f-819d-4245-a184-ac921bcb0fc0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:25:48.962674972+08:00","start_ts":450095571455705089,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095571455705089,"checkpoint_ts":450095571455705089,"checkpoint_time":"2024-05-29 18:25:45.702"} Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89e5ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47904, start at 2024-05-29 18:25:50.480939383 +0800 CST m=+4.331598919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:50.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:50.443 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:50.443 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a7bc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47157, start at 2024-05-29 18:25:30.111832814 +0800 CST m=+4.632027291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:30.118 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:30.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:30.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53295, start at 2024-05-29 18:25:06.08725242 +0800 CST m=+4.389241304 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8730080019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:53392, start at 2024-05-29 18:25:06.087211703 +0800 CST m=+4.327219458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:06.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:06.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:06.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [Wed May 29 18:25:56 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.5749757499.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d13404d1-bf3d-46c2-80b9-979a094cda66 {"id":"d13404d1-bf3d-46c2-80b9-979a094cda66","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e25e5193 d13404d1-bf3d-46c2-80b9-979a094cda66 /tidb/cdc/default/default/upstream/7374365639542525065 {"id":7374365639542525065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d13404d1-bf3d-46c2-80b9-979a094cda66 {"id":"d13404d1-bf3d-46c2-80b9-979a094cda66","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e25e5193 d13404d1-bf3d-46c2-80b9-979a094cda66 /tidb/cdc/default/default/upstream/7374365639542525065 {"id":7374365639542525065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d13404d1-bf3d-46c2-80b9-979a094cda66 {"id":"d13404d1-bf3d-46c2-80b9-979a094cda66","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e25e5193 d13404d1-bf3d-46c2-80b9-979a094cda66 /tidb/cdc/default/default/upstream/7374365639542525065 {"id":7374365639542525065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57557.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Create changefeed successfully! ID: test Info: {"upstream_id":7374365639542525065,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-29T18:25:59.78323626+08:00","start_ts":450095575116546053,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095575116546053,"checkpoint_ts":450095575116546053,"checkpoint_time":"2024-05-29 18:25:59.667"} PASS ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "workload"="core" "scanproportion"="0" "insertproportion"="0" "operationcount"="0" "recordcount"="10" "mysql.db"="changefeed_dup_error_restart" "readproportion"="0" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "threadcount"="2" "mysql.port"="4000" ********************************************** Run finished, takes 13.904208ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1349.7, Avg(us): 2566, Min(us): 1195, Max(us): 7812, 95th(us): 8000, 99th(us): 8000 [Wed May 29 18:25:50 CST 2024] <<<<<< START cdc server in changefeed_dup_error_restart case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLDuplicateEntryError=5%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.5215452156.out server --log-file /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:25:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 {"id":"aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e16c011a aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 /tidb/cdc/default/default/upstream/7374365367499251744 {"id":7374365367499251744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 {"id":"aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e16c011a aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 /tidb/cdc/default/default/upstream/7374365367499251744 {"id":7374365367499251744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 {"id":"aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e16c011a aa0f9fe8-db79-4107-af7b-d8ffb6b37bc6 /tidb/cdc/default/default/upstream/7374365367499251744 {"id":7374365367499251744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.52213.out cli changefeed create --start-ts=450095557374377985 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: f2e81bc4-3a08-413a-82fd-c608b4a241e9 Info: {"upstream_id":7374365367499251744,"namespace":"default","id":"f2e81bc4-3a08-413a-82fd-c608b4a241e9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:25:53.780070297+08:00","start_ts":450095557374377985,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095557374377985,"checkpoint_ts":450095557374377985,"checkpoint_time":"2024-05-29 18:24:51.986"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43093, start at 2024-05-29 18:25:37.533151793 +0800 CST m=+4.302525733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.540 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:25:55 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a5840008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47084, start at 2024-05-29 18:25:29.96537566 +0800 CST m=+4.574060935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:29.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:29.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:29.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f88a7bc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--71sf9, pid:47157, start at 2024-05-29 18:25:30.111832814 +0800 CST m=+4.632027291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:30.118 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:30.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:30.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:26:13 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.5107351075.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + workdir=/tmp/tidb_cdc_test/csv_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_partition_table ++ date + echo '[Wed May 29 18:26:09 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>>' [Wed May 29 18:26:09 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:26:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b61496af-760d-4fe7-84fa-8eeacae493a9 {"id":"b61496af-760d-4fe7-84fa-8eeacae493a9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2b1328f b61496af-760d-4fe7-84fa-8eeacae493a9 /tidb/cdc/default/default/upstream/7374365740786084000 {"id":7374365740786084000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b61496af-760d-4fe7-84fa-8eeacae493a9 {"id":"b61496af-760d-4fe7-84fa-8eeacae493a9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2b1328f b61496af-760d-4fe7-84fa-8eeacae493a9 /tidb/cdc/default/default/upstream/7374365740786084000 {"id":7374365740786084000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b61496af-760d-4fe7-84fa-8eeacae493a9 {"id":"b61496af-760d-4fe7-84fa-8eeacae493a9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2b1328f b61496af-760d-4fe7-84fa-8eeacae493a9 /tidb/cdc/default/default/upstream/7374365740786084000 {"id":7374365740786084000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89e5ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47904, start at 2024-05-29 18:25:50.480939383 +0800 CST m=+4.331598919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:50.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:50.443 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:50.443 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b832c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42602, start at 2024-05-29 18:26:16.943572757 +0800 CST m=+4.190463741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log table partition_table.t not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891a500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43007, start at 2024-05-29 18:25:37.446799356 +0800 CST m=+4.288033541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f891bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:43093, start at 2024-05-29 18:25:37.533151793 +0800 CST m=+4.302525733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:37.540 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:37.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:37.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.47188.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095581137731585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095581137731585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:26:22 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.4722747229.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 2-th check, retry later table partition_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:26:25 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 {"id":"1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2d2368e 1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 /tidb/cdc/default/default/upstream/7374365768250567250 {"id":7374365768250567250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 {"id":"1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2d2368e 1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 /tidb/cdc/default/default/upstream/7374365768250567250 {"id":7374365768250567250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 {"id":"1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e2d2368e 1b118b72-a6f9-4ec7-bb2d-c99bbfa07f91 /tidb/cdc/default/default/upstream/7374365768250567250 {"id":7374365768250567250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.47292.out cli changefeed create --start-ts=450095581137731585 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: b13698df-1b4c-4621-a746-71643b9500a4 Info: {"upstream_id":7374365768250567250,"namespace":"default","id":"b13698df-1b4c-4621-a746-71643b9500a4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:26:26.327730648+08:00","start_ts":450095581137731585,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095581137731585,"checkpoint_ts":450095581137731585,"checkpoint_time":"2024-05-29 18:26:22.636"} PASS table changefeed_dup_error_restart.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.port"="4000" "insertproportion"="0" "dotransactions"="false" "readproportion"="0" "workload"="core" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "recordcount"="10" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_dup_error_restart" "mysql.user"="root" "updateproportion"="0" ********************************************** Run finished, takes 5.269082ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3320.3, Avg(us): 990, Min(us): 488, Max(us): 2402, 95th(us): 3000, 99th(us): 3000 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table partition_table.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table.finish_mark not exists for 5-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/29 18:26:29.178 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_3] [numQueries=200] [2024/05/29 18:26:29.185 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_8] [numQueries=200] [2024/05/29 18:26:29.186 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_5] [numQueries=200] [2024/05/29 18:26:29.187 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_2] [numQueries=200] [2024/05/29 18:26:29.188 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_6] [numQueries=200] [2024/05/29 18:26:29.190 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_4] [numQueries=200] [2024/05/29 18:26:29.228 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_0] [numQueries=200] [2024/05/29 18:26:29.427 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_3] [2024/05/29 18:26:29.427 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_5] [2024/05/29 18:26:29.428 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_6] [2024/05/29 18:26:29.428 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_2] [2024/05/29 18:26:29.428 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_8] [2024/05/29 18:26:29.432 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_4] [2024/05/29 18:26:29.464 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_0] [2024/05/29 18:26:29.734 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_9] [numQueries=200] [2024/05/29 18:26:29.756 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_7] [numQueries=200] [2024/05/29 18:26:29.851 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_1] [numQueries=200] [2024/05/29 18:26:29.890 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_9] [2024/05/29 18:26:29.912 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_7] [2024/05/29 18:26:30.003 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_1] table test.end_mark_table not exists for 1-th check, retry later table test.end_mark_table not exists for 2-th check, retry later table test.end_mark_table not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later table test.end_mark_table not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89f244000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47812, start at 2024-05-29 18:25:51.263813857 +0800 CST m=+5.176900649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:51.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:51.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:51.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f89e5ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--36sgm, pid:47904, start at 2024-05-29 18:25:50.480939383 +0800 CST m=+4.331598919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:27:50.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:25:50.443 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:15:50.443 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:26:37 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.5200752009.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 9-th check, retry later table test.end_mark_table not exists for 5-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.end_mark_table not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:26:40 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1977220f-67d9-4429-9dea-d917e3c03658 {"id":"1977220f-67d9-4429-9dea-d917e3c03658","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e30b3d90 1977220f-67d9-4429-9dea-d917e3c03658 /tidb/cdc/default/default/upstream/7374365828596784931 {"id":7374365828596784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1977220f-67d9-4429-9dea-d917e3c03658 {"id":"1977220f-67d9-4429-9dea-d917e3c03658","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e30b3d90 1977220f-67d9-4429-9dea-d917e3c03658 /tidb/cdc/default/default/upstream/7374365828596784931 {"id":7374365828596784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1977220f-67d9-4429-9dea-d917e3c03658 {"id":"1977220f-67d9-4429-9dea-d917e3c03658","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e30b3d90 1977220f-67d9-4429-9dea-d917e3c03658 /tidb/cdc/default/default/upstream/7374365828596784931 {"id":7374365828596784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 table partition_table.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:26:42 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> table test.end_mark_table not exists for 7-th check, retry later table test.end_mark_table not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b832c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42602, start at 2024-05-29 18:26:16.943572757 +0800 CST m=+4.190463741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table test.end_mark_table not exists for 9-th check, retry later table test.end_mark_table not exists for 10-th check, retry later table test.end_mark_table exists check diff successfully + '[' 0 -eq 143 ']' + sleep 1 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:26:51 CST 2024] <<<<<< run test case processor_panic success! >>>>>> + true + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.5261752619.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57684.out cli changefeed pause -c test PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57726.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7374365639542525065,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-29T18:25:59.78323626+08:00","start_ts":450095575116546053,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095589612847110,"checkpoint_ts":450095588734664733,"checkpoint_time":"2024-05-29 18:26:51.616"} PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57764.out cli changefeed resume -c test table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... table changefeed_dup_error_restart.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:26:56 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57815.out cli changefeed pause -c test PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57849.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} Update changefeed config successfully! ID: test Info: {"upstream_id":7374365639542525065,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-29T18:25:59.78323626+08:00","start_ts":450095575116546053,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095590674530305,"checkpoint_ts":450095588865998876,"checkpoint_time":"2024-05-29 18:26:52.117"} PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.57884.out cli changefeed resume -c test PASS table test.finish_mark exists check table exists success check diff successfully coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:27:01 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b82780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42493, start at 2024-05-29 18:26:16.890532707 +0800 CST m=+4.190263890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8b832c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:42602, start at 2024-05-29 18:26:16.943572757 +0800 CST m=+4.190463741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:28:16.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:26:16.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:16:16.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed skip some DDLs in tidb v4.0.x [Wed May 29 18:27:04 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.4665746659.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:27:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5003c60e-94b7-4804-98ea-d8b634285cd8 {"id":"5003c60e-94b7-4804-98ea-d8b634285cd8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e36f7291 5003c60e-94b7-4804-98ea-d8b634285cd8 /tidb/cdc/default/default/upstream/7374365944200006697 {"id":7374365944200006697,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5003c60e-94b7-4804-98ea-d8b634285cd8 {"id":"5003c60e-94b7-4804-98ea-d8b634285cd8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e36f7291 5003c60e-94b7-4804-98ea-d8b634285cd8 /tidb/cdc/default/default/upstream/7374365944200006697 {"id":7374365944200006697,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5003c60e-94b7-4804-98ea-d8b634285cd8 {"id":"5003c60e-94b7-4804-98ea-d8b634285cd8","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e36f7291 5003c60e-94b7-4804-98ea-d8b634285cd8 /tidb/cdc/default/default/upstream/7374365944200006697 {"id":7374365944200006697,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2510 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:26:59.594","puller_resolved_ts":"2024-05-29 18:26:59.594","last_synced_ts":"2024-05-29 18:24:43.593","now_ts":"2024-05-29 18:27:00.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:26:59.594","puller_resolved_ts":"2024-05-29' '18:26:59.594","last_synced_ts":"2024-05-29' '18:24:43.593","now_ts":"2024-05-29' '18:27:00.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:26:59.594","puller_resolved_ts":"2024-05-29' '18:26:59.594","last_synced_ts":"2024-05-29' '18:24:43.593","now_ts":"2024-05-29' '18:27:00.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... ***************** properties ***************** "workload"="core" "threadcount"="10" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "fieldcount"="3" "requestdistribution"="uniform" "recordcount"="50000" "operationcount"="0" "readallfields"="true" "mysql.user"="root" "dotransactions"="false" "mysql.db"="consistent_replicate_nfs" "readproportion"="0" "updateproportion"="0" ********************************************** start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table batch_update_to_no_batch.finish_mark exists check diff successfully Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:27:14 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f658c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53367, start at 2024-05-29 18:27:20.576118654 +0800 CST m=+4.448121958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37086, start at 2024-05-29 18:27:20.980983607 +0800 CST m=+4.370415505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log INSERT - Takes(s): 10.0, Count: 5974, OPS: 598.6, Avg(us): 16658, Min(us): 1416, Max(us): 90606, 95th(us): 42000, 99th(us): 64000 Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa4340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:57086, start at 2024-05-29 18:27:24.590103878 +0800 CST m=+4.424184441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release INSERT - Takes(s): 20.0, Count: 10795, OPS: 540.3, Avg(us): 18482, Min(us): 1416, Max(us): 119229, 95th(us): 44000, 99th(us): 64000 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 2-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 1-th time, retry later check diff failed 2-th time, retry later table row_format.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58893, start at 2024-05-29 18:27:37.029179298 +0800 CST m=+4.345180005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.992 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.992 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7374365828596784931 450095600588029953 450095585881751555 run task successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check_changefeed_state http://127.0.0.1:2379 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de -s + info='{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' + echo '{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' { "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null } ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"stopped",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"stopped",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7374365828596784931 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... table row_format.finish_mark not exists for 5-th check, retry later INSERT - Takes(s): 30.0, Count: 18452, OPS: 615.5, Avg(us): 16207, Min(us): 1246, Max(us): 119229, 95th(us): 41000, 99th(us): 60000 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release run task successfully check_changefeed_state http://127.0.0.1:2379 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de -s + info='{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "normal", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' + echo '{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "normal", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' { "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "normal", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null } ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"normal",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"normal",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7374365828596784931 450095601374461953 450095601374461954 Verifying downstream PD is started... table row_format.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37086, start at 2024-05-29 18:27:20.980983607 +0800 CST m=+4.370415505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release run task successfully check_changefeed_state http://127.0.0.1:2379 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de -s + info='{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' + echo '{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null }' { "upstream_id": 7374365828596784931, "namespace": "default", "id": "6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de", "state": "stopped", "checkpoint_tso": 450095601374461954, "checkpoint_time": "2024-05-29 18:27:39.833", "error": null } ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"stopped",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de",' '"state":' '"stopped",' '"checkpoint_tso":' 450095601374461954, '"checkpoint_time":' '"2024-05-29' '18:27:39.833",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 05a4b62b-c181-43f4-864f-8a1c69cfed53 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=05a4b62b-c181-43f4-864f-8a1c69cfed53 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 05a4b62b-c181-43f4-864f-8a1c69cfed53 -s Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "05a4b62b-c181-43f4-864f-8a1c69cfed53", "state": "normal", "checkpoint_tso": 450095602881789958, "checkpoint_time": "2024-05-29 18:27:45.583", "error": null }' + echo '{ "upstream_id": 7374365828596784931, "namespace": "default", "id": "05a4b62b-c181-43f4-864f-8a1c69cfed53", "state": "normal", "checkpoint_tso": 450095602881789958, "checkpoint_time": "2024-05-29 18:27:45.583", "error": null }' { "upstream_id": 7374365828596784931, "namespace": "default", "id": "05a4b62b-c181-43f4-864f-8a1c69cfed53", "state": "normal", "checkpoint_tso": 450095602881789958, "checkpoint_time": "2024-05-29 18:27:45.583", "error": null } ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"05a4b62b-c181-43f4-864f-8a1c69cfed53",' '"state":' '"normal",' '"checkpoint_tso":' 450095602881789958, '"checkpoint_time":' '"2024-05-29' '18:27:45.583",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374365828596784931, '"namespace":' '"default",' '"id":' '"05a4b62b-c181-43f4-864f-8a1c69cfed53",' '"state":' '"normal",' '"checkpoint_tso":' 450095602881789958, '"checkpoint_time":' '"2024-05-29' '18:27:45.583",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7374365828596784931 table row_format.finish_mark exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f658c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53367, start at 2024-05-29 18:27:20.576118654 +0800 CST m=+4.448121958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:27:47 CST 2024] <<<<<< run test case row_format success! >>>>>> ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... run task successfully Changefeed remove successfully. ID: 6cf4b2dd-4dbb-48ce-b59e-9d4d4bf300de CheckpointTs: 450095601374461954 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7374365828596784931 450095603484983301 450095602881789958 450095601374461954 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa4340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:57086, start at 2024-05-29 18:27:24.590103878 +0800 CST m=+4.424184441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... run task successfully Changefeed remove successfully. ID: 05a4b62b-c181-43f4-864f-8a1c69cfed53 CheckpointTs: 450095603995901955 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7374365828596784931 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:27:51 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory INSERT - Takes(s): 40.0, Count: 29438, OPS: 736.3, Avg(us): 13561, Min(us): 900, Max(us): 119229, 95th(us): 39000, 99th(us): 58000 start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release INSERT - Takes(s): 50.0, Count: 37247, OPS: 745.2, Avg(us): 13393, Min(us): 900, Max(us): 119229, 95th(us): 38000, 99th(us): 56000 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea changefeed is working normally rts: 450095606395830300->450095606579593200 checkpoint: 450095606395830300->450095607182262300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/29 18:28:01.866 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=713.501159ms] [namespace=] [changefeed=] run task successfully resume changefeed 3750b729-47ca-4fdb-95ee-a83edf20ccea from 450095606579593229 check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37007, start at 2024-05-29 18:27:20.999349375 +0800 CST m=+4.443413067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:21.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f6c440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:37086, start at 2024-05-29 18:27:20.980983607 +0800 CST m=+4.370415505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58893, start at 2024-05-29 18:27:37.029179298 +0800 CST m=+4.345180005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.992 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.992 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.40867.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + tso='450095607819534337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095607819534337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095607819534337 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:28:04 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4090640908.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed is working normally rts: 450095607182262300->450095607706550300 checkpoint: 450095607444406300->450095607719657500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/29 18:28:03.161 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450095606579593223] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=] [changefeed=] [error="Error 1007: Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea Starting Upstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:28:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb45b5a-f364-402e-a9ec-842746b4a3d9 {"id":"6bb45b5a-f364-402e-a9ec-842746b4a3d9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e45be78e 6bb45b5a-f364-402e-a9ec-842746b4a3d9 /tidb/cdc/default/default/upstream/7374366209563803846 {"id":7374366209563803846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb45b5a-f364-402e-a9ec-842746b4a3d9 {"id":"6bb45b5a-f364-402e-a9ec-842746b4a3d9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e45be78e 6bb45b5a-f364-402e-a9ec-842746b4a3d9 /tidb/cdc/default/default/upstream/7374366209563803846 {"id":7374366209563803846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb45b5a-f364-402e-a9ec-842746b4a3d9 {"id":"6bb45b5a-f364-402e-a9ec-842746b4a3d9","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e45be78e 6bb45b5a-f364-402e-a9ec-842746b4a3d9 /tidb/cdc/default/default/upstream/7374366209563803846 {"id":7374366209563803846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095607819534337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.40969.out cli changefeed create --start-ts=450095607819534337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374366209563803846,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:28:08.123494662+08:00","start_ts":450095607819534337,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095607819534337,"checkpoint_ts":450095607819534337,"checkpoint_time":"2024-05-29 18:28:04.419"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release changefeed is working normally rts: 450095607968956400->450095608060444700 checkpoint: 450095608060444700->450095608060444700 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/29 18:28:06.240 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=86.507039ms] [namespace=] [changefeed=] run task successfully resume changefeed 3750b729-47ca-4fdb-95ee-a83edf20ccea from 450095608060444702 check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f64c4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53284, start at 2024-05-29 18:27:20.515527944 +0800 CST m=+4.475011570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8f658c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--xbn3x, pid:53367, start at 2024-05-29 18:27:20.576118654 +0800 CST m=+4.448121958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:20.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:20.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:20.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.57320.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095608998658049 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095608998658049 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa44c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:56996, start at 2024-05-29 18:27:24.590183809 +0800 CST m=+4.469888634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.563 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.563 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f8fa4340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--qrcq6, pid:57086, start at 2024-05-29 18:27:24.590103878 +0800 CST m=+4.424184441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:24.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:24.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:24.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.61348.out cli tso query --pd=http://127.0.0.1:2379 run task failed 1-th time, retry later + set +x + tso='450095609570394113 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095609570394113 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 60.0, Count: 47293, OPS: 788.5, Avg(us): 12658, Min(us): 900, Max(us): 119229, 95th(us): 37000, 99th(us): 55000 check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 1m3.588194423s INSERT - Takes(s): 63.6, Count: 50000, OPS: 786.6, Avg(us): 12634, Min(us): 900, Max(us): 119229, 95th(us): 37000, 99th(us): 54000 table consistent_replicate_nfs.usertable exists changefeed is working normally rts: 450095610065846300->450095610327990300 checkpoint: 450095610065846300->450095610590134300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/29 18:28:09.164 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450095608060444694] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) wait process cdc.test exit for 1-th time... check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ad00001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48483, start at 2024-05-29 18:28:14.318848817 +0800 CST m=+4.212866571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.324 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log changefeed is working normally rts: 450095610852278300->450095610852278300 checkpoint: 450095610734313500->450095611127529500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/29 18:28:16.384 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=80.108944ms] [namespace=] [changefeed=] run task successfully resume changefeed 3750b729-47ca-4fdb-95ee-a83edf20ccea from 450095610734313498 check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... run task failed 1-th time, retry later Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58805, start at 2024-05-29 18:27:37.030640002 +0800 CST m=+4.399343485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9066800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--cxk16, pid:58893, start at 2024-05-29 18:27:37.029179298 +0800 CST m=+4.345180005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:29:37.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:27:36.992 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:17:36.992 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Wed May 29 18:28:23 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.6297162973.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:28:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/04090c82-008f-4b32-a31d-54f058324645 {"id":"04090c82-008f-4b32-a31d-54f058324645","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e4a65d8e 04090c82-008f-4b32-a31d-54f058324645 /tidb/cdc/default/default/upstream/7374366288800486332 {"id":7374366288800486332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/04090c82-008f-4b32-a31d-54f058324645 {"id":"04090c82-008f-4b32-a31d-54f058324645","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e4a65d8e 04090c82-008f-4b32-a31d-54f058324645 /tidb/cdc/default/default/upstream/7374366288800486332 {"id":7374366288800486332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/04090c82-008f-4b32-a31d-54f058324645 {"id":"04090c82-008f-4b32-a31d-54f058324645","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e4a65d8e 04090c82-008f-4b32-a31d-54f058324645 /tidb/cdc/default/default/upstream/7374366288800486332 {"id":7374366288800486332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Wed May 29 18:28:26 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.6303363035.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed is working normally rts: 450095612700393500->450095613486825500 checkpoint: 450095612700393500->450095613486825500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/29 18:28:19.165 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450095610734313492] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "450095610316980252" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/29 18:28:26.800 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/29 18:28:26.800 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/29 18:28:26.801 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450095609792430084] [resolvedTs=450095610316980252] [2024/05/29 18:28:26.801 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=374.25ยตs] [2024/05/29 18:28:26.802 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:28:26.802 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:28:26.823 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=22.164591ms] [2024/05/29 18:28:26.831 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:28:26.884 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/29 18:28:26.887 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/29 18:28:26.887 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/29 18:28:26.888 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/29 18:28:26.932 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/29 18:28:26.942 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/29 18:28:27.160 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=82] [checkpointTs=450095610316980252] [2024/05/29 18:28:27.160 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=82] [checkpointTs=450095610316980252] [2024/05/29 18:28:27.265 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1334] [appliedDDLCount=0] [currentCheckpoint=450095610316980252] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/29 18:28:27.265 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/29 18:28:27.266 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/29 18:28:27.266 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/29 18:28:27.266 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/29 18:28:27.266 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] Apply redo log successfully check diff successfully [Wed May 29 18:28:27 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 48973 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6927 0 --:--:-- --:--:-- --:--:-- 6951 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:28:04.419","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-29 18:28:28.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:28:04.419","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:28:28.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:28:04.419","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:28:28.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster changefeed is working normally rts: 450095613749231600->450095614273257500 checkpoint: 450095613749231600->450095614273257500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/29 18:28:28.564 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=160.671581ms] [namespace=] [changefeed=] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:28:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93d2f9be-acaf-42a2-94d2-b2f6208d64d3 {"id":"93d2f9be-acaf-42a2-94d2-b2f6208d64d3","address":"127.0.0.1:8400","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118fc3e4a6925c 93d2f9be-acaf-42a2-94d2-b2f6208d64d3 /tidb/cdc/default/default/upstream/7374366293021313862 {"id":7374366293021313862,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93d2f9be-acaf-42a2-94d2-b2f6208d64d3 {"id":"93d2f9be-acaf-42a2-94d2-b2f6208d64d3","address":"127.0.0.1:8400","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118fc3e4a6925c 93d2f9be-acaf-42a2-94d2-b2f6208d64d3 /tidb/cdc/default/default/upstream/7374366293021313862 {"id":7374366293021313862,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93d2f9be-acaf-42a2-94d2-b2f6208d64d3 {"id":"93d2f9be-acaf-42a2-94d2-b2f6208d64d3","address":"127.0.0.1:8400","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118fc3e4a6925c 93d2f9be-acaf-42a2-94d2-b2f6208d64d3 /tidb/cdc/default/default/upstream/7374366293021313862 {"id":7374366293021313862,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x resume changefeed 3750b729-47ca-4fdb-95ee-a83edf20ccea from 450095613788291086 check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... run task failed 1-th time, retry later check_ts_forward 3750b729-47ca-4fdb-95ee-a83edf20ccea Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release changefeed is working normally rts: 450095615584239600->450095616108265500 checkpoint: 450095615846121500->450095616108265500 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/29 18:28:31.356 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=145.022325ms] [namespace=] [changefeed=] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:28:37 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ad00001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48483, start at 2024-05-29 18:28:14.318848817 +0800 CST m=+4.212866571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.324 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f90c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41937, start at 2024-05-29 18:28:51.91407908 +0800 CST m=+4.323583026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.920 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.907 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.907 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9580580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50140, start at 2024-05-29 18:29:00.591657915 +0800 CST m=+4.447978972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.598 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Wed May 29 18:29:04 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.6147061472.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:07 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b57be44d-bb8c-4496-8fdc-7ab47669259d {"id":"b57be44d-bb8c-4496-8fdc-7ab47669259d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e47118b3 b57be44d-bb8c-4496-8fdc-7ab47669259d /tidb/cdc/default/default/upstream/7374366238937823717 {"id":7374366238937823717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b57be44d-bb8c-4496-8fdc-7ab47669259d {"id":"b57be44d-bb8c-4496-8fdc-7ab47669259d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e47118b3 b57be44d-bb8c-4496-8fdc-7ab47669259d /tidb/cdc/default/default/upstream/7374366238937823717 {"id":7374366238937823717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b57be44d-bb8c-4496-8fdc-7ab47669259d {"id":"b57be44d-bb8c-4496-8fdc-7ab47669259d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e47118b3 b57be44d-bb8c-4496-8fdc-7ab47669259d /tidb/cdc/default/default/upstream/7374366238937823717 {"id":7374366238937823717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.61527.out cli changefeed create --start-ts=450095609570394113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7374366238937823717,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:29:07.403213519+08:00","start_ts":450095609570394113,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095609570394113,"checkpoint_ts":450095609570394113,"checkpoint_time":"2024-05-29 18:28:11.098"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.simple not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095623411335203, "checkpoint_time": "2024-05-29 18:29:03.897", "error": null }' + echo '{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095623411335203, "checkpoint_time": "2024-05-29 18:29:03.897", "error": null }' { "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095623411335203, "checkpoint_time": "2024-05-29 18:29:03.897", "error": null } ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095623411335203, '"checkpoint_time":' '"2024-05-29' '18:29:03.897",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095623411335203, '"checkpoint_time":' '"2024-05-29' '18:29:03.897",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ac4c001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48389, start at 2024-05-29 18:28:14.262487411 +0800 CST m=+4.240209362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f92ad00001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--pzj92, pid:48483, start at 2024-05-29 18:28:14.318848817 +0800 CST m=+4.212866571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:14.324 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:14.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:14.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.52563.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095625816244225 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095625816244225 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:29:13 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.5260652608.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.61879.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b08eb716-f069-4735-984d-7da9e431101d {"id":"b08eb716-f069-4735-984d-7da9e431101d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5661d8e b08eb716-f069-4735-984d-7da9e431101d /tidb/cdc/default/default/upstream/7374366447224395367 {"id":7374366447224395367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b08eb716-f069-4735-984d-7da9e431101d {"id":"b08eb716-f069-4735-984d-7da9e431101d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5661d8e b08eb716-f069-4735-984d-7da9e431101d /tidb/cdc/default/default/upstream/7374366447224395367 {"id":7374366447224395367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b08eb716-f069-4735-984d-7da9e431101d {"id":"b08eb716-f069-4735-984d-7da9e431101d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5661d8e b08eb716-f069-4735-984d-7da9e431101d /tidb/cdc/default/default/upstream/7374366447224395367 {"id":7374366447224395367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.52663.out cli changefeed create --start-ts=450095625816244225 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 92719cb1-d885-4835-8cf0-a54739958741 Info: {"upstream_id":7374366447224395367,"namespace":"default","id":"92719cb1-d885-4835-8cf0-a54739958741","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:29:16.766443112+08:00","start_ts":450095625816244225,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095625816244225,"checkpoint_ts":450095625816244225,"checkpoint_time":"2024-05-29 18:29:13.071"} PASS ***************** properties ***************** "dotransactions"="false" "operationcount"="0" "mysql.db"="changefeed_error" "threadcount"="4" "readallfields"="true" "workload"="core" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.user"="root" "scanproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "updateproportion"="0" "recordcount"="20" "readproportion"="0" ********************************************** Run finished, takes 11.207028ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3238.8, Avg(us): 2108, Min(us): 1112, Max(us): 5058, 95th(us): 5000, 99th(us): 6000 [Wed May 29 18:29:06 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5744357445.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c {"id":"98dd0db7-c7f0-45f9-8241-f3b46712227c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c419 98dd0db7-c7f0-45f9-8241-f3b46712227c /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c {"id":"98dd0db7-c7f0-45f9-8241-f3b46712227c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c419 98dd0db7-c7f0-45f9-8241-f3b46712227c /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c {"id":"98dd0db7-c7f0-45f9-8241-f3b46712227c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c419 98dd0db7-c7f0-45f9-8241-f3b46712227c /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.57502.out cli changefeed create --start-ts=450095608998658049 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7374366211526050341,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:29:09.762413947+08:00","start_ts":450095608998658049,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049,"checkpoint_time":"2024-05-29 18:28:08.917"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450095608998658049, "checkpoint_time": "2024-05-29 18:28:08.917", "error": { "time": "2024-05-29T18:29:09.823644517+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at 450095608998658049" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450095608998658049, "checkpoint_time": "2024-05-29 18:28:08.917", "error": { "time": "2024-05-29T18:29:09.823644517+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at 450095608998658049" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450095608998658049, "checkpoint_time": "2024-05-29 18:28:08.917", "error": { "time": "2024-05-29T18:29:09.823644517+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at 450095608998658049" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450095608998658049, '"checkpoint_time":' '"2024-05-29' '18:28:08.917",' '"error":' '{' '"time":' '"2024-05-29T18:29:09.823644517+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at '450095608998658049"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450095608998658049, '"checkpoint_time":' '"2024-05-29' '18:28:08.917",' '"error":' '{' '"time":' '"2024-05-29T18:29:09.823644517+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at '450095608998658049"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at 450095608998658049' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450095608998657749 is earlier than or equal to GC safepoint at 450095608998658049 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.57596.out cli changefeed resume -c changefeed-error PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table changefeed_error.usertable not exists for 1-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff successfully ***************** properties ***************** "mysql.port"="4000" "updateproportion"="0" "insertproportion"="0" "workload"="core" "recordcount"="20" "mysql.db"="changefeed_error" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "threadcount"="4" "dotransactions"="false" "scanproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** Run finished, takes 6.923945ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3753.8, Avg(us): 1107, Min(us): 727, Max(us): 2520, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"98dd0db7-c7f0-45f9-8241-f3b46712227c","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + [[ /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f90c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41937, start at 2024-05-29 18:28:51.91407908 +0800 CST m=+4.323583026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.920 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.907 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.907 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null }' + echo '{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null }' { "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null } ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450095625797107713, '"checkpoint_time":' '"2024-05-29' '18:29:12.998",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450095625797107713, '"checkpoint_time":' '"2024-05-29' '18:29:12.998",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.61971.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + [[ /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7374366238937823717,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:29:07.403213519+08:00","start_ts":450095609570394113,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095625797107713,"checkpoint_ts":450095625797107713,"checkpoint_time":"2024-05-29 18:29:12.998"} PASS start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... coverage: 3.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62010.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null }' + echo '{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null }' { "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095625797107713, "checkpoint_time": "2024-05-29 18:29:12.998", "error": null } ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095625797107713, '"checkpoint_time":' '"2024-05-29' '18:29:12.998",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095625797107713, '"checkpoint_time":' '"2024-05-29' '18:29:12.998",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62093.out cli changefeed --changefeed-id custom-changefeed-name remove check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + [[ /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/98dd0db7-c7f0-45f9-8241-f3b46712227c + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 450095628156403714 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62173.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7374366238937823717,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:29:26.453077972+08:00","start_ts":450095629296467970,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095629296467970,"checkpoint_ts":450095629296467970,"checkpoint_time":"2024-05-29 18:29:26.347"} PASS table bdr_mode.t1 exists table bdr_mode.t1 exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.63179.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374366288800486332,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-29T18:29:22.626061302+08:00","start_ts":450095628296650754,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095628296650754,"checkpoint_ts":450095628296650754,"checkpoint_time":"2024-05-29 18:29:22.533"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.63214.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Create changefeed successfully! ID: test-2 Info: {"upstream_id":7374366293021313862,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-29T18:29:23.215501516+08:00","start_ts":450095628442140679,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095628442140679,"checkpoint_ts":450095628442140679,"checkpoint_time":"2024-05-29 18:29:23.088"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Wed May 29 18:29:28 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5783657838.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9580580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50140, start at 2024-05-29 18:29:00.591657915 +0800 CST m=+4.447978972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.598 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095629296467970, "checkpoint_time": "2024-05-29 18:29:26.347", "error": null }' + echo '{ "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095629296467970, "checkpoint_time": "2024-05-29 18:29:26.347", "error": null }' { "upstream_id": 7374366238937823717, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450095629296467970, "checkpoint_time": "2024-05-29 18:29:26.347", "error": null } ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095629296467970, '"checkpoint_time":' '"2024-05-29' '18:29:26.347",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374366238937823717, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450095629296467970, '"checkpoint_time":' '"2024-05-29' '18:29:26.347",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62303.out cli unsafe delete-service-gc-safepoint cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:29:31 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77827373-4e31-44e9-9b9f-2a310bf17eeb {"id":"77827373-4e31-44e9-9b9f-2a310bf17eeb","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c48b 77827373-4e31-44e9-9b9f-2a310bf17eeb /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374366211526050341,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:29:09.762413947+08:00","start-ts":450095608998658049,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-29T18:29:28.561139942+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095629878165505} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450095626182197253,"min-table-barrier-ts":450095626182197253,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77827373-4e31-44e9-9b9f-2a310bf17eeb {"id":"77827373-4e31-44e9-9b9f-2a310bf17eeb","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c48b 77827373-4e31-44e9-9b9f-2a310bf17eeb /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374366211526050341,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:29:09.762413947+08:00","start-ts":450095608998658049,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-29T18:29:28.561139942+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095629878165505} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450095626182197253,"min-table-barrier-ts":450095626182197253,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77827373-4e31-44e9-9b9f-2a310bf17eeb {"id":"77827373-4e31-44e9-9b9f-2a310bf17eeb","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c48b 77827373-4e31-44e9-9b9f-2a310bf17eeb /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374366211526050341,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-29T18:29:09.762413947+08:00","start-ts":450095608998658049,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-29T18:29:28.561139942+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g91afb776d","epoch":450095629878165505} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450095626182197253,"min-table-barrier-ts":450095626182197253,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450095626182197253, "checkpoint_time": "2024-05-29 18:29:14.467", "error": { "time": "2024-05-29T18:29:28.561139942+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450095626182197253, "checkpoint_time": "2024-05-29 18:29:14.467", "error": { "time": "2024-05-29T18:29:28.561139942+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450095626182197253, "checkpoint_time": "2024-05-29 18:29:14.467", "error": { "time": "2024-05-29T18:29:28.561139942+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450095626182197253, '"checkpoint_time":' '"2024-05-29' '18:29:14.467",' '"error":' '{' '"time":' '"2024-05-29T18:29:28.561139942+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450095626182197253, '"checkpoint_time":' '"2024-05-29' '18:29:14.467",' '"error":' '{' '"time":' '"2024-05-29T18:29:28.561139942+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.57938.out cli changefeed remove -c changefeed-error coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62336.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x Changefeed remove successfully. ID: changefeed-error CheckpointTs: 450095626182197253 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Wed May 29 18:29:33 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5803458036.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62447.out cli unsafe resolve-lock --region=90 PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f6c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41845, start at 2024-05-29 18:28:51.763132001 +0800 CST m=+4.260563728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f94f90c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:41937, start at 2024-05-29 18:28:51.91407908 +0800 CST m=+4.323583026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:30:51.920 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:28:51.907 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:18:51.907 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.45760.out cli tso query --pd=http://127.0.0.1:2379 coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.62487.out cli unsafe resolve-lock --region=90 --ts=450095630817165315 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + tso='450095631759048705 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095631759048705 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095631759048705 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:29:36 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4579345795.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 127k 3696 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:36 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351d6033-2e8f-495f-a0d2-f46dae487d7b {"id":"351d6033-2e8f-495f-a0d2-f46dae487d7b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c4ae 351d6033-2e8f-495f-a0d2-f46dae487d7b /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351d6033-2e8f-495f-a0d2-f46dae487d7b {"id":"351d6033-2e8f-495f-a0d2-f46dae487d7b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c4ae 351d6033-2e8f-495f-a0d2-f46dae487d7b /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351d6033-2e8f-495f-a0d2-f46dae487d7b {"id":"351d6033-2e8f-495f-a0d2-f46dae487d7b","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c4ae 351d6033-2e8f-495f-a0d2-f46dae487d7b /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58086.out cli changefeed create --start-ts=450095608998658049 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7374366211526050341,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:29:36.556720232+08:00","start_ts":450095608998658049,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049,"checkpoint_time":"2024-05-29 18:28:08.917"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1591 0 --:--:-- --:--:-- --:--:-- 1592 100 86 100 86 0 0 1588 0 --:--:-- --:--:-- --:--:-- 1563 + info='{"state":"normal","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049}' + echo '{"state":"normal","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049}' {"state":"normal","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049} ++ echo '{"state":"normal","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96ef97c8-6085-47f7-a0b3-c5f9c0de9853 {"id":"96ef97c8-6085-47f7-a0b3-c5f9c0de9853","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5c4578f 96ef97c8-6085-47f7-a0b3-c5f9c0de9853 /tidb/cdc/default/default/upstream/7374366612042267371 {"id":7374366612042267371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96ef97c8-6085-47f7-a0b3-c5f9c0de9853 {"id":"96ef97c8-6085-47f7-a0b3-c5f9c0de9853","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5c4578f 96ef97c8-6085-47f7-a0b3-c5f9c0de9853 /tidb/cdc/default/default/upstream/7374366612042267371 {"id":7374366612042267371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96ef97c8-6085-47f7-a0b3-c5f9c0de9853 {"id":"96ef97c8-6085-47f7-a0b3-c5f9c0de9853","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5c4578f 96ef97c8-6085-47f7-a0b3-c5f9c0de9853 /tidb/cdc/default/default/upstream/7374366612042267371 {"id":7374366612042267371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095631759048705 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.45852.out cli changefeed create --start-ts=450095631759048705 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374366612042267371,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:29:39.457851218+08:00","start_ts":450095631759048705,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095631759048705,"checkpoint_ts":450095631759048705,"checkpoint_time":"2024-05-29 18:29:35.741"} PASS } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 210k 0 --:--:-- --:--:-- --:--:-- 171k { "version": "v6.5.9-12-g91afb776d", "git_hash": "91afb776d8e607224e8f53cc50086ec346799afe", "id": "757f7d20-b593-4356-9f19-a7fbdd43d4cd", "pid": 61475, "is_owner": true coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5245 0 --:--:-- --:--:-- --:--:-- 5261 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:29:35.741","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-29 18:29:39.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:29:35.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:29:39.000","info":"Data' syncing is 'finished"}' ++ jq .synced }cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:29:39 CST 2024] <<<<<< run test case cli success! >>>>>> + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:29:35.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:29:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-29 18:29:35.741' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:29:35.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:29:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:29:35.741","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:29:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-29 18:29:39' + echo 'sink_checkpoint_ts is 2024-05-29' 18:29:35.741 sink_checkpoint_ts is 2024-05-29 18:29:35.741 ++ date -d '2024-05-29 18:29:35.741' +%s + checkpoint_timestamp=1716978575 ++ date -d '2024-05-29 18:29:39' +%s + current_timestamp=1716978579 + '[' 4 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2576 0 --:--:-- --:--:-- --:--:-- 2581 + info='{"state":"warning","resolved_ts":450095623875330057,"checkpoint_ts":450095623875330057,"last_warning":{"time":"2024-05-29T18:29:38.727695945+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":450095623875330057,"checkpoint_ts":450095623875330057,"last_warning":{"time":"2024-05-29T18:29:38.727695945+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":450095623875330057,"checkpoint_ts":450095623875330057,"last_warning":{"time":"2024-05-29T18:29:38.727695945+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":450095623875330057,"checkpoint_ts":450095623875330057,"last_warning":{"time":"2024-05-29T18:29:38.727695945+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":450095623875330057,"checkpoint_ts":450095623875330057,"last_warning":{"time":"2024-05-29T18:29:38.727695945+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58127.out cli changefeed remove -c changefeed-error-1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 450095623875330057 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Wed May 29 18:29:41 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5817158173.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:44 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 {"id":"c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c506 c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 {"id":"c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c506 c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 {"id":"c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c506 c85a1d27-7ed8-4f0e-b9c0-c3473f9b74d1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58232.out cli changefeed create --start-ts=450095608998658049 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7374366211526050341,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:29:44.823728547+08:00","start_ts":450095608998658049,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095608998658049,"checkpoint_ts":450095608998658049,"checkpoint_time":"2024-05-29 18:28:08.917"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:29:44.830500566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:29:44.830500566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-29T18:29:44.830500566+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-29T18:29:44.830500566+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-29T18:29:44.830500566+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450095608998658049 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58314.out cli changefeed remove -c changefeed-error-2 Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:29:46 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5834758349.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f957dec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50054, start at 2024-05-29 18:29:00.438039436 +0800 CST m=+4.380770200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0f9580580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--mmqgl, pid:50140, start at 2024-05-29 18:29:00.591657915 +0800 CST m=+4.447978972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:31:00.598 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:29:00.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:19:00.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.54074.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095634994429953 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095634994429953 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:29:48 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.5410954111.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:49 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2aea3988-993f-4360-94ba-652218da5da1 {"id":"2aea3988-993f-4360-94ba-652218da5da1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c52d 2aea3988-993f-4360-94ba-652218da5da1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2aea3988-993f-4360-94ba-652218da5da1 {"id":"2aea3988-993f-4360-94ba-652218da5da1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c52d 2aea3988-993f-4360-94ba-652218da5da1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2aea3988-993f-4360-94ba-652218da5da1 {"id":"2aea3988-993f-4360-94ba-652218da5da1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e468c52d 2aea3988-993f-4360-94ba-652218da5da1 /tidb/cdc/default/default/upstream/7374366211526050341 {"id":7374366211526050341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58402.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/conf/enable_redo.toml Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7374366211526050341,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:29:50.355385279+08:00","start_ts":450095635566952450,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/changefeed_error/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095635566952450,"checkpoint_ts":450095635566952450,"checkpoint_time":"2024-05-29 18:29:50.267"} PASS [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450095635566952450, '"checkpoint_time":' '"2024-05-29' '18:29:50.267",' '"error":' '{' '"time":' '"2024-05-29T18:29:50.452482179+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:29:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/443df691-98fc-4858-8823-bb50da3ed617 {"id":"443df691-98fc-4858-8823-bb50da3ed617","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5f4578f 443df691-98fc-4858-8823-bb50da3ed617 /tidb/cdc/default/default/upstream/7374366646644015399 {"id":7374366646644015399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/443df691-98fc-4858-8823-bb50da3ed617 {"id":"443df691-98fc-4858-8823-bb50da3ed617","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5f4578f 443df691-98fc-4858-8823-bb50da3ed617 /tidb/cdc/default/default/upstream/7374366646644015399 {"id":7374366646644015399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/443df691-98fc-4858-8823-bb50da3ed617 {"id":"443df691-98fc-4858-8823-bb50da3ed617","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3e5f4578f 443df691-98fc-4858-8823-bb50da3ed617 /tidb/cdc/default/default/upstream/7374366646644015399 {"id":7374366646644015399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.54171.out cli changefeed create --start-ts=450095634994429953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 665f2a58-36e3-4520-b1d8-9543d34b236d Info: {"upstream_id":7374366646644015399,"namespace":"default","id":"665f2a58-36e3-4520-b1d8-9543d34b236d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:29:51.809052106+08:00","start_ts":450095634994429953,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095634994429953,"checkpoint_ts":450095634994429953,"checkpoint_time":"2024-05-29 18:29:48.083"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450095635566952450, "checkpoint_time": "2024-05-29 18:29:50.267", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450095635566952450, '"checkpoint_time":' '"2024-05-29' '18:29:50.267",' '"error":' '{' '"time":' '"2024-05-29T18:29:50.452482179+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58569.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": { "time": "2024-05-29T18:29:50.452482179+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' '{' '"time":' '"2024-05-29T18:29:50.452482179+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' '{' '"time":' '"2024-05-29T18:29:50.452482179+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58648.out cli changefeed resume -c changefeed-initialize-error PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null }' + echo '{ "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null }' { "upstream_id": 7374366211526050341, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450095637191983108, "checkpoint_time": "2024-05-29 18:29:56.466", "error": null } ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374366211526050341, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450095637191983108, '"checkpoint_time":' '"2024-05-29' '18:29:56.466",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.58726.out cli changefeed remove -c changefeed-initialize-error Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 450095637191983108 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:30:00 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> table savepoint.finish_mark not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } table savepoint.finish_mark not exists for 2-th check, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table savepoint.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:30:16 CST 2024] <<<<<< run test case savepoint success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 16489 0 --:--:-- --:--:-- --:--:-- 17357 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:30:36.541","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-29 18:30:33.941","now_ts":"2024-05-29 18:30:38.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:30:36.541","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:30:33.941","now_ts":"2024-05-29' '18:30:38.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:30:36.541","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:30:33.941","now_ts":"2024-05-29' '18:30:38.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table ddl_sequence.finish_mark not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_sequence.finish_mark not exists for 4-th check, retry later table ddl_sequence.finish_mark not exists for 5-th check, retry later table ddl_sequence.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:30:58 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5782 0 --:--:-- --:--:-- --:--:-- 5815 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:32:46.641","puller_resolved_ts":"2024-05-29 18:32:41.691","last_synced_ts":"2024-05-29 18:30:33.941","now_ts":"2024-05-29 18:32:49.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:32:46.641","puller_resolved_ts":"2024-05-29' '18:32:41.691","last_synced_ts":"2024-05-29' '18:30:33.941","now_ts":"2024-05-29' '18:32:49.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 44831 7.2 0.0 13218236 151876 ? Sl 18:29 0:14 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 44870 4.1 0.0 12839172 141560 ? Sl 18:29 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-29T18:32:58.416+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:32:58.416+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:32:58.511+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:04.418+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:04.418+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:04.512+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:08.403+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:33:08.403+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-29T18:33:08.406+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:33:08.406+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-29T18:33:08.497+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-29T18:33:08.497+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716978788.5139909,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001ac6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1716978788.5140636,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716978788.5156813,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001a7a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1716978788.5157585,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1716978788.5965688,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0019fc1c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1716978788.5966244,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:33:10.418+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:10.419+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:10.513+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:33:16.420+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:16.420+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:16.514+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:33:22.421+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:22.422+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:22.516+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:33:28.422+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:28.424+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:28.517+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-29T18:33:34.423+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0001cd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:34.425+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001028000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-29T18:33:34.518+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ab9dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c0640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:47062, start at 2024-05-29 18:33:59.609484191 +0800 CST m=+4.294469863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c0640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:47062, start at 2024-05-29 18:33:59.609484191 +0800 CST m=+4.294469863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c084000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:46990, start at 2024-05-29 18:33:59.60056918 +0800 CST m=+4.341324599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.607 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fa7c0640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:47062, start at 2024-05-29 18:33:59.609484191 +0800 CST m=+4.294469863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:35:59.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:33:59.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:23:59.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.50897.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095712282607617 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095712282607617 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095712282607617 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:34:43 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5092450926.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:34:46 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31d2f97e-532c-4f3e-a1b8-d6390ba7176d {"id":"31d2f97e-532c-4f3e-a1b8-d6390ba7176d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ea72778e 31d2f97e-532c-4f3e-a1b8-d6390ba7176d /tidb/cdc/default/default/upstream/7374367929323577703 {"id":7374367929323577703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31d2f97e-532c-4f3e-a1b8-d6390ba7176d {"id":"31d2f97e-532c-4f3e-a1b8-d6390ba7176d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ea72778e 31d2f97e-532c-4f3e-a1b8-d6390ba7176d /tidb/cdc/default/default/upstream/7374367929323577703 {"id":7374367929323577703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31d2f97e-532c-4f3e-a1b8-d6390ba7176d {"id":"31d2f97e-532c-4f3e-a1b8-d6390ba7176d","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ea72778e 31d2f97e-532c-4f3e-a1b8-d6390ba7176d /tidb/cdc/default/default/upstream/7374367929323577703 {"id":7374367929323577703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095712282607617 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.50976.out cli changefeed create --start-ts=450095712282607617 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374367929323577703,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:34:46.609081397+08:00","start_ts":450095712282607617,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095712282607617,"checkpoint_ts":450095712282607617,"checkpoint_time":"2024-05-29 18:34:42.914"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 50047 6.9 0.3 3157308 1437276 ? Sl 18:34 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 50048 7.1 0.3 3158328 1444008 ? Sl 18:34 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 50049 7.6 0.3 3172668 1456704 ? Sl 18:34 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 50051 7.0 0.3 3155768 1447512 ? Sl 18:34 0:04 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6487 0 --:--:-- --:--:-- --:--:-- 6567 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:35:45.814","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-29 18:35:40.764","now_ts":"2024-05-29 18:35:47.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:35:45.814","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:35:40.764","now_ts":"2024-05-29' '18:35:47.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:35:45.814","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:35:40.764","now_ts":"2024-05-29' '18:35:47.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 27547 0 --:--:-- --:--:-- --:--:-- 27807 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:35:46.814","puller_resolved_ts":"2024-05-29 18:35:46.814","last_synced_ts":"2024-05-29 18:35:40.764","now_ts":"2024-05-29 18:37:57.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:35:46.814","puller_resolved_ts":"2024-05-29' '18:35:46.814","last_synced_ts":"2024-05-29' '18:35:40.764","now_ts":"2024-05-29' '18:37:57.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:35:46.814","puller_resolved_ts":"2024-05-29' '18:35:46.814","last_synced_ts":"2024-05-29' '18:35:40.764","now_ts":"2024-05-29' '18:37:57.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52095, start at 2024-05-29 18:38:19.142550266 +0800 CST m=+4.639312098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52095, start at 2024-05-29 18:38:19.142550266 +0800 CST m=+4.639312098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52030, start at 2024-05-29 18:38:19.142771684 +0800 CST m=+4.726695700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fb798000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:52095, start at 2024-05-29 18:38:19.142550266 +0800 CST m=+4.639312098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:40:19.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:38:19.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:28:19.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.55888.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095780694065153 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095780694065153 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095780694065153 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:39:04 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5592355925.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:39:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efccce5f-4db3-4384-9a7c-2c49f09ea245 {"id":"efccce5f-4db3-4384-9a7c-2c49f09ea245","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ee6b8490 efccce5f-4db3-4384-9a7c-2c49f09ea245 /tidb/cdc/default/default/upstream/7374369041429567229 {"id":7374369041429567229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efccce5f-4db3-4384-9a7c-2c49f09ea245 {"id":"efccce5f-4db3-4384-9a7c-2c49f09ea245","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ee6b8490 efccce5f-4db3-4384-9a7c-2c49f09ea245 /tidb/cdc/default/default/upstream/7374369041429567229 {"id":7374369041429567229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efccce5f-4db3-4384-9a7c-2c49f09ea245 {"id":"efccce5f-4db3-4384-9a7c-2c49f09ea245","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3ee6b8490 efccce5f-4db3-4384-9a7c-2c49f09ea245 /tidb/cdc/default/default/upstream/7374369041429567229 {"id":7374369041429567229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095780694065153 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.55965.out cli changefeed create --start-ts=450095780694065153 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374369041429567229,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:39:07.571287963+08:00","start_ts":450095780694065153,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095780694065153,"checkpoint_ts":450095780694065153,"checkpoint_time":"2024-05-29 18:39:03.883"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 55802 1.3 0.0 1064148 140020 ? Sl 18:39 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716979141469286953.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 55811 1.0 0.0 1146076 118176 ? Sl 18:39 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716979141473926501.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 55844 1.2 0.0 1499624 138300 ? Sl 18:39 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1716979141531613109.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6541 0 --:--:-- --:--:-- --:--:-- 6567 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:40:04.733","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-29 18:40:00.732","now_ts":"2024-05-29 18:40:07.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:40:04.733","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:40:00.732","now_ts":"2024-05-29' '18:40:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:40:04.733","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-29' '18:40:00.732","now_ts":"2024-05-29' '18:40:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 8345 0 --:--:-- --:--:-- --:--:-- 8500 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-29 18:42:14.882","puller_resolved_ts":"2024-05-29 18:42:07.882","last_synced_ts":"2024-05-29 18:40:00.732","now_ts":"2024-05-29 18:42:17.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:42:14.882","puller_resolved_ts":"2024-05-29' '18:42:07.882","last_synced_ts":"2024-05-29' '18:40:00.732","now_ts":"2024-05-29' '18:42:17.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-29' '18:42:14.882","puller_resolved_ts":"2024-05-29' '18:42:07.882","last_synced_ts":"2024-05-29' '18:40:00.732","now_ts":"2024-05-29' '18:42:17.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc7612c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57117, start at 2024-05-29 18:42:37.787137984 +0800 CST m=+4.489957881 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc7612c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57117, start at 2024-05-29 18:42:37.787137984 +0800 CST m=+4.489957881 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc75fb80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57044, start at 2024-05-29 18:42:37.691454116 +0800 CST m=+4.471504736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fc7612c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:57117, start at 2024-05-29 18:42:37.787137984 +0800 CST m=+4.489957881 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:44:37.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:42:37.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:32:37.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.60980.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095848572059649 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095848572059649 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450095848572059649 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Wed May 29 18:43:23 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.6101961021.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:43:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7176511b-52a2-45df-809a-7e54e1b283ca {"id":"7176511b-52a2-45df-809a-7e54e1b283ca","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f25efa90 7176511b-52a2-45df-809a-7e54e1b283ca /tidb/cdc/default/default/upstream/7374370159944351254 {"id":7374370159944351254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7176511b-52a2-45df-809a-7e54e1b283ca {"id":"7176511b-52a2-45df-809a-7e54e1b283ca","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f25efa90 7176511b-52a2-45df-809a-7e54e1b283ca /tidb/cdc/default/default/upstream/7374370159944351254 {"id":7374370159944351254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7176511b-52a2-45df-809a-7e54e1b283ca {"id":"7176511b-52a2-45df-809a-7e54e1b283ca","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f25efa90 7176511b-52a2-45df-809a-7e54e1b283ca /tidb/cdc/default/default/upstream/7374370159944351254 {"id":7374370159944351254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450095848572059649 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.61076.out cli changefeed create --start-ts=450095848572059649 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374370159944351254,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:43:26.586967121+08:00","start_ts":450095848572059649,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095848572059649,"checkpoint_ts":450095848572059649,"checkpoint_time":"2024-05-29 18:43:22.817"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8047 0 --:--:-- --:--:-- --:--:-- 8123 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-29 18:43:22.817","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-29 18:43:46.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:43:22.817","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:43:46.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-29' '18:43:22.817","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-29' '18:43:46.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Wed May 29 18:43:56 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Wed May 29 18:43:56 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd2eb00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62156, start at 2024-05-29 18:44:12.88055348 +0800 CST m=+4.266989529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd2eb00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62156, start at 2024-05-29 18:44:12.88055348 +0800 CST m=+4.266989529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd30500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62074, start at 2024-05-29 18:44:12.98702048 +0800 CST m=+4.421223023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.999 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.999 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fcd2eb00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:62156, start at 2024-05-29 18:44:12.88055348 +0800 CST m=+4.266989529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:46:12.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:44:12.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:34:12.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.66069.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095887990915073 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095887990915073 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:45:53 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.6610966111.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:45:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33befab7-fe98-4ab5-99fb-9b6cbefebed1 {"id":"33befab7-fe98-4ab5-99fb-9b6cbefebed1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f3c5ad10 33befab7-fe98-4ab5-99fb-9b6cbefebed1 /tidb/cdc/default/default/upstream/7374370556141552332 {"id":7374370556141552332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33befab7-fe98-4ab5-99fb-9b6cbefebed1 {"id":"33befab7-fe98-4ab5-99fb-9b6cbefebed1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f3c5ad10 33befab7-fe98-4ab5-99fb-9b6cbefebed1 /tidb/cdc/default/default/upstream/7374370556141552332 {"id":7374370556141552332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33befab7-fe98-4ab5-99fb-9b6cbefebed1 {"id":"33befab7-fe98-4ab5-99fb-9b6cbefebed1","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f3c5ad10 33befab7-fe98-4ab5-99fb-9b6cbefebed1 /tidb/cdc/default/default/upstream/7374370556141552332 {"id":7374370556141552332,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.66180.out cli changefeed create --start-ts=450095887990915073 '--sink-uri=mysql://syncpoint:dGVzdDEyMzQ1Ng==@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 5a218017-4483-4ca2-a868-f30b4c66fdfa Info: {"upstream_id":7374370556141552332,"namespace":"default","id":"5a218017-4483-4ca2-a868-f30b4c66fdfa","sink_uri":"mysql://syncpoint:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-29T18:45:57.171478384+08:00","start_ts":450095887990915073,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450095887990915073,"checkpoint_ts":450095887990915073,"checkpoint_time":"2024-05-29 18:45:53.188"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "scanproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "readproportion"="0" "threadcount"="2" "recordcount"="10" "mysql.db"="testSync" "workload"="core" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" "updateproportion"="0" ********************************************** Run finished, takes 641.999449ms INSERT - Takes(s): 0.5, Count: 10, OPS: 18.6, Avg(us): 128306, Min(us): 5203, Max(us): 233649, 95th(us): 234000, 99th(us): 234000 ***************** properties ***************** "mysql.port"="4000" "workload"="core" "threadcount"="2" "mysql.db"="testSync" "dotransactions"="false" "mysql.user"="root" "recordcount"="10" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "scanproportion"="0" "insertproportion"="0" "readproportion"="0" ********************************************** Run finished, takes 4.648046ms INSERT - Takes(s): 0.0, Count: 9, OPS: 3000.7, Avg(us): 771, Min(us): 534, Max(us): 1509, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "workload"="core" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "threadcount"="2" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "recordcount"="10" "readproportion"="0" "mysql.user"="root" "mysql.db"="testSync" "scanproportion"="0" ********************************************** Run finished, takes 5.000773ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2800.3, Avg(us): 854, Min(us): 551, Max(us): 1915, 95th(us): 2000, 99th(us): 2000 table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists ____________________________________ *************************** 1. row *************************** primary_ts: 450095887990915073 secondary_ts: 450095891265355781 *************************** 2. row *************************** primary_ts: 450095895855235072 secondary_ts: 450095896128126977 *************************** 3. row *************************** primary_ts: 450095903719555072 secondary_ts: 450095903992709121 *************************** 4. row *************************** primary_ts: 450095911583875072 secondary_ts: 450095911882981377 *************************** 5. row *************************** primary_ts: 450095919448195072 secondary_ts: 450095919747301377 *************************** 6. row *************************** primary_ts: 450095927312515072 secondary_ts: 450095927611621377 *************************** 7. row *************************** primary_ts: 450095935176835072 secondary_ts: 450095935502155777 *************************** 8. row *************************** primary_ts: 450095943041155072 secondary_ts: 450095943366475777 *************************** 9. row *************************** primary_ts: 450095950905475072 secondary_ts: 450095950968913921 skip invalid syncpoint primary_ts: 450095887990915073, first_ddl_ts: 450095893010972680 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:50:05 CST 2024] <<<<<< run test case syncpoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe440fc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68329, start at 2024-05-29 18:50:30.852890365 +0800 CST m=+4.254822597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe440fc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68329, start at 2024-05-29 18:50:30.852890365 +0800 CST m=+4.254822597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe441b8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68259, start at 2024-05-29 18:50:30.923964895 +0800 CST m=+4.398497507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0fe440fc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:68329, start at 2024-05-29 18:50:30.852890365 +0800 CST m=+4.254822597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:52:30.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:50:30.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:40:30.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.72255.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450095972272046081 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450095972272046081 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:51:15 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.7229772299.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:51:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15ceb615-3b68-4816-ac00-fd818cd6bcd0 {"id":"15ceb615-3b68-4816-ac00-fd818cd6bcd0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f9927991 15ceb615-3b68-4816-ac00-fd818cd6bcd0 /tidb/cdc/default/default/upstream/7374372189506086657 {"id":7374372189506086657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15ceb615-3b68-4816-ac00-fd818cd6bcd0 {"id":"15ceb615-3b68-4816-ac00-fd818cd6bcd0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f9927991 15ceb615-3b68-4816-ac00-fd818cd6bcd0 /tidb/cdc/default/default/upstream/7374372189506086657 {"id":7374372189506086657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15ceb615-3b68-4816-ac00-fd818cd6bcd0 {"id":"15ceb615-3b68-4816-ac00-fd818cd6bcd0","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3f9927991 15ceb615-3b68-4816-ac00-fd818cd6bcd0 /tidb/cdc/default/default/upstream/7374372189506086657 {"id":7374372189506086657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1564 --:--:-- --:--:-- --:--:-- 1560 100 142 0 0 100 142 0 1563 --:--:-- --:--:-- --:--:-- 1560 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading go.uber.org/zap v1.19.1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 build mysql_test successfully run all mysql test cases time="2024-05-29T18:51:37+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-29T18:51:37+08:00" level=warning msg="Create new db&{0 0xc000126000 0 {0 0} [0xc00012c090] map[] 0 1 0xc00012a000 false map[0xc00012c090:map[0xc00012c090:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/alter_table.test: ok! 70 test cases passed, take time 20.476851003 s time="2024-05-29T18:52:31+08:00" level=info msg="run test [alter_table] ok" time="2024-05-29T18:52:31+08:00" level=warning msg="Create new db&{0 0xc000014328 0 {0 0} [0xc00038c000] map[] 0 1 0xc000038c60 false map[0xc00038c000:map[0xc00038c000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/bigint.test: ok! 138 test cases passed, take time 12.924814345 s time="2024-05-29T18:52:44+08:00" level=info msg="run test [bigint] ok" time="2024-05-29T18:52:44+08:00" level=warning msg="Create new db&{0 0xc000394190 0 {0 0} [0xc00046e000] map[] 0 1 0xc000300780 false map[0xc00046e000:map[0xc00046e000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/composite_index.test: ok! 12 test cases passed, take time 1.5552929789999999 s time="2024-05-29T18:52:46+08:00" level=info msg="run test [composite_index] ok" time="2024-05-29T18:52:46+08:00" level=warning msg="Create new db&{0 0xc000394218 0 {0 0} [0xc000212bd0] map[] 0 1 0xc000300ae0 false map[0xc000212bd0:map[0xc000212bd0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.03029593 s time="2024-05-29T18:52:48+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-29T18:52:48+08:00" level=warning msg="Create new db&{0 0xc0002aa078 0 {0 0} [0xc0005b2120] map[] 0 1 0xc00030e0c0 false map[0xc0005b2120:map[0xc0005b2120:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/date_formats.test: ok! 99 test cases passed, take time 5.355296819 s time="2024-05-29T18:52:54+08:00" level=info msg="run test [date_formats] ok" time="2024-05-29T18:52:54+08:00" level=warning msg="Create new db&{0 0xc0005a43f8 0 {0 0} [0xc00046e2d0] map[] 0 1 0xc0005a90e0 false map[0xc00046e2d0:map[0xc00046e2d0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/drop.test: ok! 48 test cases passed, take time 8.727002115 s time="2024-05-29T18:53:03+08:00" level=info msg="run test [drop] ok" time="2024-05-29T18:53:03+08:00" level=warning msg="Create new db&{0 0xc0000141e8 0 {0 0} [0xc000124000] map[] 0 1 0xc000300a80 false map[0xc000124000:map[0xc000124000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.053600254 s time="2024-05-29T18:53:05+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later table test.finish_mark not exists for 9-th check, retry later table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later table test.finish_mark not exists for 12-th check, retry later table test.finish_mark not exists for 13-th check, retry later table test.finish_mark not exists for 14-th check, retry later table test.finish_mark not exists for 15-th check, retry later table test.finish_mark not exists for 16-th check, retry later table test.finish_mark not exists for 17-th check, retry later table test.finish_mark not exists for 18-th check, retry later table test.finish_mark not exists for 19-th check, retry later table test.finish_mark not exists for 20-th check, retry later table test.finish_mark not exists for 21-th check, retry later table test.finish_mark not exists for 22-th check, retry later table test.finish_mark not exists for 23-th check, retry later table test.finish_mark not exists for 24-th check, retry later table test.finish_mark not exists for 25-th check, retry later table test.finish_mark not exists for 26-th check, retry later table test.finish_mark not exists for 27-th check, retry later table test.finish_mark not exists for 28-th check, retry later table test.finish_mark not exists for 29-th check, retry later table test.finish_mark not exists for 30-th check, retry later table test.finish_mark not exists for 31-th check, retry later table test.finish_mark not exists for 32-th check, retry later table test.finish_mark not exists for 33-th check, retry later table test.finish_mark not exists for 34-th check, retry later table test.finish_mark not exists for 35-th check, retry later table test.finish_mark not exists for 36-th check, retry later table test.finish_mark not exists for 37-th check, retry later table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:54:20 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b210000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:75034, start at 2024-05-29 18:54:43.859604526 +0800 CST m=+4.423520230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.866 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b210000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:75034, start at 2024-05-29 18:54:43.859604526 +0800 CST m=+4.423520230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.866 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-16-gd9c5bf64d0 Edition: Community Git Commit Hash: d9c5bf64d0dd123180a60642337956face1c1e3d Git Branch: release-6.5 UTC Build Time: 2024-05-28 10:13:53 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b2140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:74956, start at 2024-05-29 18:54:43.87187783 +0800 CST m=+4.488987641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f0ff3b210000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-252--2g660, pid:75034, start at 2024-05-29 18:54:43.859604526 +0800 CST m=+4.423520230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240529-18:56:43.866 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240529-18:54:43.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240529-18:44:43.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.78874.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450096038632488961 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450096038632488961 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Wed May 29 18:55:28 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.7890878910.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Wed, 29 May 2024 10:55:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f27768f7-99a6-4005-a7c0-a5a14665deb3 {"id":"f27768f7-99a6-4005-a7c0-a5a14665deb3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3fd6d5c8d f27768f7-99a6-4005-a7c0-a5a14665deb3 /tidb/cdc/default/default/upstream/7374373269273762430 {"id":7374373269273762430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f27768f7-99a6-4005-a7c0-a5a14665deb3 {"id":"f27768f7-99a6-4005-a7c0-a5a14665deb3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3fd6d5c8d f27768f7-99a6-4005-a7c0-a5a14665deb3 /tidb/cdc/default/default/upstream/7374373269273762430 {"id":7374373269273762430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f27768f7-99a6-4005-a7c0-a5a14665deb3 {"id":"f27768f7-99a6-4005-a7c0-a5a14665deb3","address":"127.0.0.1:8300","version":"v6.5.9-12-g91afb776d"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc3fd6d5c8d f27768f7-99a6-4005-a7c0-a5a14665deb3 /tidb/cdc/default/default/upstream/7374373269273762430 {"id":7374373269273762430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.78982.out cli changefeed create --start-ts=450096038632488961 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 18117595-3836-4a76-b442-e01036b3a77f Info: {"upstream_id":7374373269273762430,"namespace":"default","id":"18117595-3836-4a76-b442-e01036b3a77f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-29T18:55:31.543170174+08:00","start_ts":450096038632488961,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g91afb776d","resolved_ts":450096038632488961,"checkpoint_ts":450096038632488961,"checkpoint_time":"2024-05-29 18:55:27.840"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Wed May 29 18:56:29 CST 2024] <<<<<< run test case tiflash success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-252/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS