Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-6.5/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-1jc1x-8f758 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "781dbe7cfd94613fdb3072a09355b4a47cfdafb3" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-1jc1x" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" First time build. Skipping changelog. [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 20 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=8ef91400-4bf2-4b25-9849-f227364309c2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-1jc1x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-de6e69a8ea75a42ea3ef5bc05e88104f5785c12a567deb6ce7101124d13f2fcf NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-1jc1x pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE="" GOARCH="amd64" GOBIN="" GOCACHE="/home/jenkins/.cache/go-build" GOENV="/home/jenkins/.config/go/env" GOEXE="" GOEXPERIMENT="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="linux" GOINSECURE="" GOMODCACHE="/go/pkg/mod" GONOPROXY="" GONOSUMDB="" GOOS="linux" GOPATH="/go" GOPRIVATE="" GOPROXY="http://goproxy.apps.svc,https://proxy.golang.org,direct" GOROOT="/usr/local/go" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" GOVCS="" GOVERSION="go1.19.13" GCCGO="gccgo" GOAMD64="v1" AR="ar" CC="gcc" CXX="g++" CGO_ENABLED="1" GOMOD="/dev/null" GOWORK="" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build46470422=/tmp/go-build -gno-record-gcc-switches" + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--qnvcj bash [Pipeline] container [Pipeline] { [Pipeline] sh > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-list --no-walk 44e9fdd1b388d71f07c1fc94e6fccf31c512143e # timeout=10 + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 4571 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: be687b4f22ba8174 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 13 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri May 31 08:38:23 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10946/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10946/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10946/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10946/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/entry/mounter.go, cdc/entry/mounter_test.go] [Pipeline] echo diff file not matched: cdc/entry/mounter.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-350a2fb-e46df29) 190230016 bytes in 0.71 secs (269104966 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at e46df29f4 auo adjust scheduler config POST git-upload-pack (686 bytes) POST git-upload-pack (1023 bytes) POST git-upload-pack (gzip 2073 to 1091 bytes) From https://github.com/pingcap/tiflow * [new branch] release-6.5 -> origin/release-6.5 * [new ref] refs/pull/10946/head -> origin/pr/10946/head * [new tag] v6.5.0 -> v6.5.0 * [new tag] v6.5.1 -> v6.5.1 * [new tag] v6.5.2 -> v6.5.2 * [new tag] v6.5.3 -> v6.5.3 * [new tag] v6.5.4 -> v6.5.4 * [new tag] v6.5.5 -> v6.5.5 * [new tag] v6.5.6 -> v6.5.6 * [new tag] v6.5.7 -> v6.5.7 * [new tag] v6.5.8 -> v6.5.8 * [new tag] v6.5.9 -> v6.5.9 Previous HEAD position was e46df29f4 auo adjust scheduler config HEAD is now at 6277d9a8c kvClient (ticdc): revert e5999e3 to remove useless metrics (#11184) 🚧 Checkouting to base SHA:6277d9a8c9e9b43613b6d78e2af8aca1fde9593f... HEAD is now at 6277d9a8c kvClient (ticdc): revert e5999e3 to remove useless metrics (#11184) ✅ Checked. 🎉 🧾 HEAD info: 6277d9a8c9e9b43613b6d78e2af8aca1fde9593f 6277d9a8c kvClient (ticdc): revert e5999e3 to remove useless metrics (#11184) 54e93ed06 syncpoint (ticdc): make syncpoint support base64 encoded password (#11162) 0ba932965 (redo)ticdc: fix the event orderliness in redo log (#11117) (#11180) 🚧 Pre-merge heads of pull requests to base SHA: 6277d9a8c9e9b43613b6d78e2af8aca1fde9593f ... Merge made by the 'ort' strategy. cdc/entry/mounter.go | 9 +++++++++ cdc/entry/mounter_test.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) 🧾 Pre-merged result: 7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7 7b0579a42 Merge commit '1b2b17ca406d5b7d993b0a238f16791becd777cb' into HEAD 1b2b17ca4 fix conflicts. 6277d9a8c kvClient (ticdc): revert e5999e3 to remove useless metrics (#11184) ✅ Pre merged 🎉 ✅ ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-6277d9a-1b2b17c already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + chmod +x ../scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh + /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh Download binaries from branch release-6.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 923 0 --:--:-- --:--:-- --:--:-- 931 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 613 0 --:--:-- --:--:-- --:--:-- 621 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2727 0 --:--:-- --:--:-- --:--:-- 2928 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 799 0 --:--:-- --:--:-- --:--:-- 803 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1/centos7/tidb-server.tar.gz 2024-05-31 16:38:54 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1/centos7/tidb-server.tar.gz [501269018/501269018] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/33d839bdd7ed98fca9823febbddb43fcdc03a36a/centos7/pd-server.tar.gz 2024-05-31 16:39:08 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/33d839bdd7ed98fca9823febbddb43fcdc03a36a/centos7/pd-server.tar.gz [140293259/140293259] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/627d4a572b4d7d523370e5ed21ee67fee8d798a6/centos7/tikv-server.tar.gz 2024-05-31 16:39:43 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/627d4a572b4d7d523370e5ed21ee67fee8d798a6/centos7/tikv-server.tar.gz [723214974/723214974] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/61348e865b1b4b46013233aa3e210ea158a364ce/centos7/tiflash.tar.gz 2024-05-31 16:39:57 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/61348e865b1b4b46013233aa3e210ea158a364ce/centos7/tiflash.tar.gz [205615551/205615551] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-31 16:40:00 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-31 16:40:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-31 16:40:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-31 16:40:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-31 16:40:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] total 1.4G drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:40 . drwxr-sr-x 18 jenkins jenkins 4.0K May 31 16:40 .. -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 31 16:40 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 31 16:40 jq lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 170M May 31 16:15 tidb-server -rwxr-xr-x 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x 1 jenkins jenkins 304M May 10 15:52 tikv-server Download SUCCESS + ls -alh ./bin/ total 1.4G drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:40 . drwxr-sr-x 18 jenkins jenkins 4.0K May 31 16:40 .. -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 31 16:40 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 31 16:40 jq lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 170M May 31 16:15 tidb-server -rwxr-xr-x 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x 1 jenkins jenkins 304M May 10 15:52 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/minio + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.4G drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:40 . drwxr-sr-x 3 jenkins jenkins 4.0K May 31 16:40 .. -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 31 16:40 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 31 16:40 jq lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 29 15:02 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 29 15:02 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 29 15:02 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 29 14:41 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 74M May 29 14:56 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 36M May 17 10:32 pd-ctl -rwxr-xr-x 1 jenkins jenkins 20M May 17 10:32 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 34M May 17 10:32 pd-recover -rwxr-xr-x 1 jenkins jenkins 95M May 17 10:31 pd-server -rwxr-xr-x 1 jenkins jenkins 23M May 17 10:32 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 27M May 17 10:32 regions-dump -rwxr-xr-x 1 jenkins jenkins 22M May 17 10:32 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 170M May 31 16:15 tidb-server -rwxr-xr-x 1 jenkins jenkins 297M May 29 15:02 tiflash -rwxr-xr-x 1 jenkins jenkins 304M May 10 15:52 tikv-server + ./bin/tidb-server -V Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 + ./bin/tikv-server -V TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:40 . drwxr-sr-x 18 jenkins jenkins 4.0K May 31 16:40 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:40:11" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc ./cmd/cdc/main.go go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/zap v1.23.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/google/uuid v1.3.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/cenkalti/backoff/v4 v4.0.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/jsonreference v0.20.0 go: downloading github.com/go-openapi/spec v0.20.6 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/stathat/consistent v1.0.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/go-openapi/jsonpointer v0.19.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/go-openapi/swag v0.21.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.24.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading github.com/go-logr/logr v1.2.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:42:54" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:43:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:43:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/chaos-mesh/go-sqlsmith v0.0.0-20220905074648-403033efad45 go: downloading github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading gorm.io/gorm v1.23.8 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/glebarez/sqlite v1.4.6 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/glebarez/go-sqlite v1.17.3 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.2.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.16.8 go: downloading modernc.org/sqlite v1.17.3 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/rs/cors v1.7.0 go: downloading modernc.org/memory v1.1.1 go: downloading modernc.org/mathutil v1.4.1 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:43:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v6.5.9-12-g7b0579a42" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 08:43:02" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.19.13 linux/amd64"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 746M drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:46 . drwxr-sr-x 18 jenkins jenkins 4.0K May 31 16:40 .. -rwxr-xr-x 1 jenkins jenkins 162M May 31 16:46 cdc -rwxr-xr-x 1 jenkins jenkins 311M May 31 16:46 cdc.test -rwxr-xr-x 1 jenkins jenkins 137M May 31 16:43 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 137M May 31 16:43 cdc_storage_consumer + ./bin/cdc version Release Version: v6.5.9-12-g7b0579a42 Git Commit Hash: 7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7 Git Branch: HEAD UTC Build Time: 2024-05-31 08:43:02 Go Version: go version go1.19.13 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-6277d9a-1b2b17c) 781311488 bytes in 50.87 secs (15359139 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x 2 jenkins jenkins 4.0K May 31 16:47 . drwxr-sr-x 18 jenkins jenkins 4.0K May 31 16:40 .. -rwxr-xr-x 1 jenkins jenkins 162M May 31 16:46 cdc -rwxr-xr-x 1 jenkins jenkins 311M May 31 16:46 cdc.test -rwxr-xr-x 1 jenkins jenkins 137M May 31 16:43 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 137M May 31 16:43 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 17M May 31 16:47 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 31 16:47 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 31 16:47 jq lrwxrwxrwx 1 jenkins jenkins 13 May 31 16:47 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 31 16:47 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 31 16:47 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 31 16:47 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 31 16:47 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 31 16:47 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 31 16:47 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 74M May 31 16:47 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 31 16:47 minio -rwxr-xr-x 1 jenkins jenkins 36M May 31 16:47 pd-ctl -rwxr-xr-x 1 jenkins jenkins 20M May 31 16:47 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 34M May 31 16:47 pd-recover -rwxr-xr-x 1 jenkins jenkins 95M May 31 16:47 pd-server -rwxr-xr-x 1 jenkins jenkins 23M May 31 16:47 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 27M May 31 16:47 regions-dump -rwxr-xr-x 1 jenkins jenkins 22M May 31 16:47 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 31 16:47 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 170M May 31 16:47 tidb-server -rwxr-xr-x 1 jenkins jenkins 297M May 31 16:47 tiflash -rwxr-xr-x 1 jenkins jenkins 304M May 31 16:47 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 48.71 secs (49018994 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G0') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G1') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G2') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G3') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G4') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G5') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G6') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G7') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G8') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G9') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G0') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G1') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G2') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G3') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G4') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G5') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G6') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G7') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G8') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G9') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-b82lt-s8f5k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3859eaab974a13cdd9a3d6245438b1bd3b6b8a9e" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-b82lt" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 [Pipeline] { [Pipeline] checkout The recommended git tool is: git Still waiting to schedule task Waiting for next available executor on ‘pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7’ Still waiting to schedule task ‘pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t’ is offline Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-8jxr6-t0mbm Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-35p90-r19z7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" kind: "Pod" metadata: annotations: runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" jenkins/label-digest: "a68c1c78eb2479f38a98d6502da5c47c7789ea7a" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-35p90" jenkins/label-digest: "98152ae859bc3db4f2d2f9a2cd5b713431c688fb" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-8jxr6" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7" namespace: "jenkins-tiflow" spec: name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: operator: "In" values: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: - "amd64" containers: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" resources: limits: readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: memory: "16Gi" cpu: "6" requests: memory: "12Gi" limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" memory: "128Mi" cpu: "100m" tty: true volumeMounts: - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - mountPath: "/home/jenkins/agent" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh" - name: "JENKINS_NAME" name: "workspace-volume" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: name: "report" resources: limits: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" memory: "256Mi" cpu: "100m" tty: true volumeMounts: readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false name: "workspace-volume" - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-tl5d3’ are offline [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7c4f63b8; decorates RemoteLauncher[hudson.remoting.Channel@5b9f579b:JNLP4-connect connection from 10.233.68.197/10.233.68.197:57508] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-3pgc6-hj96d --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "00f6c88aa69721da479e291c483847e464db866c" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-3pgc6" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@40c675b5; decorates RemoteLauncher[hudson.remoting.Channel@2b3b9edc:JNLP4-connect connection from 10.233.68.162/10.233.68.162:48282] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53beecd7; decorates RemoteLauncher[hudson.remoting.Channel@66491408:JNLP4-connect connection from 10.233.72.120/10.233.72.120:51280] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-0b1d9’ are offline No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4dac70a5; decorates RemoteLauncher[hudson.remoting.Channel@6fc85571:JNLP4-connect connection from 10.233.72.191/10.233.72.191:43464] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-xq76s’ are offline Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-s18hd’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-08n2h’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-ws0dr’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-39q36’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-234hg’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-dnm30’ are offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-006kw’ are offline Still waiting to schedule task ‘pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2’ is offline Still waiting to schedule task All nodes of label ‘pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-vpkwz’ are offline Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 6.78 secs (352229802 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G5 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group5: consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=52b534d9-7b9f-40b0-9e3c-3c3d683ee68b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G5 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-b82lt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-b82lt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 6.28 secs (380430282 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:51:32.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:51:32.953 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c25240017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:1006, start at 2024-05-31 16:49:33.046812225 +0800 CST m=+4.504752746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:51:33.055 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:33.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:33.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 6.72 secs (355503251 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 16.57 secs (144122192 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G3 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group3: changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=fcbba8a1-c8aa-4597-b1c8-4edf89ccb61a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G3 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-8jxr6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-8jxr6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G0 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group0: api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=f44464a9-e2e4-4a36-86fa-eeb8e43a20f5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G0 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-35p90 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-35p90 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G1 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group1: big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=0e2f1831-86d5-4737-b3e3-b4830d75fac1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G1 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-3pgc6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-3pgc6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:12.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:12.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e918c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:979, start at 2024-05-31 16:50:12.721947294 +0800 CST m=+4.528609950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:12.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:13.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:13.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9d040004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:1083, start at 2024-05-31 16:50:13.444961562 +0800 CST m=+4.709654945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:13.452 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.441 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.441 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:14.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:14.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabd4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:1037, start at 2024-05-31 16:50:14.400172392 +0800 CST m=+4.520088025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:14.406 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.389 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.389 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/Shopify/sarama v1.38.1 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading golang.org/x/sync v0.5.0 go: downloading golang.org/x/time v0.2.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading cloud.google.com/go v0.105.0 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading golang.org/x/sys v0.14.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 Starting CDC state checker... go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/stretchr/objx v0.5.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading google.golang.org/api v0.103.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-lzdm7-79nrn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "323041eb61734a6e25b9caf7f6e884b9e31a35c6" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-lzdm7" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@eda346b; decorates RemoteLauncher[hudson.remoting.Channel@529cc7fe:JNLP4-connect connection from 10.233.67.43/10.233.67.43:54312] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 16.50 secs (144688690 bytes/sec) [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G14 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group14: sequence simple sink_hang sink_retry sorter split_region Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=8e8d4795-6c63-42e4-bc8c-d3cd36091e48 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-lzdm7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-lzdm7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:38.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:38.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:988, start at 2024-05-31 16:51:38.492378418 +0800 CST m=+4.387981230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:38.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:32.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:32.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c25240017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:1006, start at 2024-05-31 16:49:33.046812225 +0800 CST m=+4.504752746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:33.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:33.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:33.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/google/btree v1.1.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:32.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c23ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:922, start at 2024-05-31 16:49:32.946689535 +0800 CST m=+4.483517564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:32.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:32.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:32.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375c25240017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:1006, start at 2024-05-31 16:49:33.046812225 +0800 CST m=+4.504752746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:52:33.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:49:33.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:39:33.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:52:00 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2384023842.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:52:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea72e195-4258-462a-9f26-e45bfa681962 {"id":"ea72e195-4258-462a-9f26-e45bfa681962","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9268e90 ea72e195-4258-462a-9f26-e45bfa681962 /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea72e195-4258-462a-9f26-e45bfa681962 {"id":"ea72e195-4258-462a-9f26-e45bfa681962","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9268e90 ea72e195-4258-462a-9f26-e45bfa681962 /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ea72e195-4258-462a-9f26-e45bfa681962 {"id":"ea72e195-4258-462a-9f26-e45bfa681962","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9268e90 ea72e195-4258-462a-9f26-e45bfa681962 /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9d040004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:1083, start at 2024-05-31 16:50:13.444961562 +0800 CST m=+4.709654945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.476 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.441 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.441 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e918c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:979, start at 2024-05-31 16:50:12.721947294 +0800 CST m=+4.528609950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' Verifying downstream PD is started... start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabd4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:1037, start at 2024-05-31 16:50:14.400172392 +0800 CST m=+4.520088025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.389 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.389 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9b640016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:977, start at 2024-05-31 16:50:13.362052395 +0800 CST m=+4.723860454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.337 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.337 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9d040004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:1083, start at 2024-05-31 16:50:13.444961562 +0800 CST m=+4.709654945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:13.476 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:13.441 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:13.441 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Verifying downstream PD is started... Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e9248000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:881, start at 2024-05-31 16:50:12.765220997 +0800 CST m=+4.623317287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375e918c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:979, start at 2024-05-31 16:50:12.721947294 +0800 CST m=+4.528609950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:12.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:12.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:12.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.25761.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139407574368257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139407574368257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-08n2h-3qrht Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-006kw-8vqxm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "db6b5ae374b08115150d1ee2b5e3029231ce771d" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-08n2h" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false --- apiVersion: "v1" kind: "Pod" metadata: annotations: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false jenkins/label-digest: "754abebf90fed37248feefbd42802b875a81eb69" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-006kw" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853" - name: "JENKINS_NAME" memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-0b1d9-xp0ws --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "39c6e065e5dbc28fecd7c855afee21f9987345ff" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-0b1d9" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@41f50a72; decorates RemoteLauncher[hudson.remoting.Channel@31d82260:JNLP4-connect connection from 10.233.72.241/10.233.72.241:57976] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@57d6472c; decorates RemoteLauncher[hudson.remoting.Channel@47afd313:JNLP4-connect connection from 10.233.72.250/10.233.72.250:49008] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1ef6cc29; decorates RemoteLauncher[hudson.remoting.Channel@8dfec8:JNLP4-connect connection from 10.233.107.202/10.233.107.202:40308] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:930, start at 2024-05-31 16:50:14.423756002 +0800 CST m=+4.635202472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f375eabd4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:1037, start at 2024-05-31 16:50:14.400172392 +0800 CST m=+4.520088025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:53:14.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:50:14.389 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:40:14.389 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:52:54 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2493624938.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] withEnv > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-vpkwz-cstb4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "94602d535bdaecece08d65d708bedd9a88c342b3" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-vpkwz" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:52:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.25005.out cli tso query --pd=http://127.0.0.1:2379 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-234hg-nn2fb Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-tl5d3-hmm45 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" --- apiVersion: "v1" kind: "Pod" metadata: annotations: runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ba8690f831622961c8f75328086515c4b745d51a" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-tl5d3" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1f20e9ae6145b1d149fb39d5a041448075e1e679" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-234hg" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0" - name: "JENKINS_NAME" - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test + set +x + tso='450139410323996675 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139410323996675 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:52:58 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2503825040.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:52:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ec89d2a-14c3-4466-93a3-ed448509acc5 {"id":"0ec89d2a-14c3-4466-93a3-ed448509acc5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9f1d28e 0ec89d2a-14c3-4466-93a3-ed448509acc5 /tidb/cdc/default/default/upstream/7375083364181303092 {"id":7375083364181303092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.25067.out cli changefeed create --start-ts=450139410323996675 '--sink-uri=mysql://normal:123456@127.0.0.1:3306?transaction-atomicity=none' Create changefeed successfully! ID: 233114f4-91d6-4ebb-a2fc-26861daae090 Info: {"upstream_id":7375083364181303092,"namespace":"default","id":"233114f4-91d6-4ebb-a2fc-26861daae090","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306?transaction-atomicity=none","create_time":"2024-05-31T16:52:58.437324182+08:00","start_ts":450139410323996675,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139410323996675,"checkpoint_ts":450139410323996675,"checkpoint_time":"2024-05-31 16:52:57.713"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "fieldcount"="100" "readproportion"="0" "threadcount"="10" "readallfields"="true" "mysql.host"="127.0.0.1" "scanproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "mysql.port"="4000" "dotransactions"="false" "workload"="core" "recordcount"="5000" "mysql.db"="consistent_replicate_storage_file" "updateproportion"="0" "mysql.user"="root" ********************************************** Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-ws0dr-hfj9v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "59e505c3868a2249a65892aac0fb27235e1f4827" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-ws0dr" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test INSERT - Takes(s): 9.9, Count: 1354, OPS: 136.3, Avg(us): 70604, Min(us): 2016, Max(us): 3001411, 95th(us): 185000, 99th(us): 630000 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-s18hd-08h34 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "34403aa9935960079ee0c37e9f6d9bfd9ae8f780" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-s18hd" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-dnm30-phl84 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-39q36-5cx3d --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6e6437f1dd400f229155e6abaed86ae9c45ea3f3" jenkins/label-digest: "c9a95a8801f1c00ce5bd2ca31c9d5eb2a7e870d1" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-39q36" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-dnm30" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k" namespace: "jenkins-tiflow" spec: affinity: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: operator: "In" values: - "amd64" containers: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" memory: "16Gi" cpu: "6" requests: memory: "12Gi" resources: limits: memory: "128Mi" cpu: "100m" tty: true cpu: "4" tty: true volumeMounts: volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" memory: "128Mi" cpu: "100m" tty: true volumeMounts: readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8" - name: "JENKINS_NAME" memory: "256Mi" cpu: "100m" tty: true volumeMounts: value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k" - name: "JENKINS_NAME" volumes: - emptyDir: medium: "" name: "workspace-volume" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test INSERT - Takes(s): 19.9, Count: 2885, OPS: 144.7, Avg(us): 68444, Min(us): 1821, Max(us): 3001411, 95th(us): 196000, 99th(us): 630000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 26.25 secs (90970509 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Run finished, takes 25.193286444s INSERT - Takes(s): 25.1, Count: 5000, OPS: 199.0, Avg(us): 49611, Min(us): 1821, Max(us): 3001411, 95th(us): 168000, 99th(us): 417000 [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G13 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group13: processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=f632f3bd-1a27-4f62-b631-7ad2e49813b5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-006kw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-006kw pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-xq76s-9mtf3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d33d79bfc3cb7ebe54ce3ad5f0bc7f13615e5d5b" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-xq76s" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later [Pipeline] cache Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@648d6735; decorates RemoteLauncher[hudson.remoting.Channel@44d42167:JNLP4-connect connection from 10.233.108.194/10.233.108.194:59844] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@61a84121; decorates RemoteLauncher[hudson.remoting.Channel@30870741:JNLP4-connect connection from 10.233.73.23/10.233.73.23:46988] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@21116b03; decorates RemoteLauncher[hudson.remoting.Channel@68799d97:JNLP4-connect connection from 10.233.73.216/10.233.73.216:54342] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@67b17bbc; decorates RemoteLauncher[hudson.remoting.Channel@3ad8c789:JNLP4-connect connection from 10.233.72.189/10.233.72.189:51334] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1e2d3e53; decorates RemoteLauncher[hudson.remoting.Channel@6b22cb24:JNLP4-connect connection from 10.233.127.159/10.233.127.159:33680] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@51eb5488; decorates RemoteLauncher[hudson.remoting.Channel@1ac45f15:JNLP4-connect connection from 10.233.127.83/10.233.127.83:50780] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@892e8ea; decorates RemoteLauncher[hudson.remoting.Channel@16839397:JNLP4-connect connection from 10.233.127.52/10.233.127.52:57266] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Verifying downstream PD is started... table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 7.21 secs (331086964 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:38.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:38.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b279c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:1045, start at 2024-05-31 16:53:38.957232437 +0800 CST m=+4.670104670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:38.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.969 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.969 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 3.28 secs (728872189 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G8 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group8: generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=8789673c-999f-4fb2-837d-2108faab66e8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G8 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-08n2h GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-08n2h pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G4 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group4: clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=06ea964f-4b48-451b-9e2c-0efa01c06032 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G4 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-0b1d9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-0b1d9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] withEnv [Pipeline] { Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/klauspost/compress v1.15.14 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/golang/protobuf v1.5.2 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container go: downloading github.com/docker/go-units v0.4.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Fri May 31 16:53:43 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.2587725879.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 [Pipeline] { (Test) go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7fe31066; decorates RemoteLauncher[hudson.remoting.Channel@306ae33c:JNLP4-connect connection from 10.233.66.252/10.233.66.252:57832] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { [Pipeline] { go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da [Pipeline] { [Pipeline] { Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:53:46 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a3f63cf-427d-4572-9039-41c8dbdd1528 {"id":"7a3f63cf-427d-4572-9039-41c8dbdd1528","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9d6b003 7a3f63cf-427d-4572-9039-41c8dbdd1528 /tidb/cdc/default/default/upstream/7375083358543311898 {"id":7375083358543311898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a3f63cf-427d-4572-9039-41c8dbdd1528 {"id":"7a3f63cf-427d-4572-9039-41c8dbdd1528","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9d6b003 7a3f63cf-427d-4572-9039-41c8dbdd1528 /tidb/cdc/default/default/upstream/7375083358543311898 {"id":7375083358543311898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a3f63cf-427d-4572-9039-41c8dbdd1528 {"id":"7a3f63cf-427d-4572-9039-41c8dbdd1528","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9d6b003 7a3f63cf-427d-4572-9039-41c8dbdd1528 /tidb/cdc/default/default/upstream/7375083358543311898 {"id":7375083358543311898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.25939.out cli changefeed create --start-ts=450139407574368257 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7375083358543311898,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T16:53:47.076766364+08:00","start_ts":450139407574368257,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139407574368257,"checkpoint_ts":450139407574368257,"checkpoint_time":"2024-05-31 16:52:47.224"} PASS Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:988, start at 2024-05-31 16:51:38.492378418 +0800 CST m=+4.387981230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.570 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" + info='{ "upstream_id": 7375083358543311898, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T16:53:47.082630012+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7375083358543311898, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T16:53:47.082630012+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7375083358543311898, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T16:53:47.082630012+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375083358543311898, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-31T16:53:47.082630012+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375083358543311898, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-31T16:53:47.082630012+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Fri May 31 16:53:38 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.2386123863.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:53:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bde3c208-9ab8-43b0-83b6-e85ccfc53e9b {"id":"bde3c208-9ab8-43b0-83b6-e85ccfc53e9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9be2d0a bde3c208-9ab8-43b0-83b6-e85ccfc53e9b /tidb/cdc/default/default/upstream/7375083358042639761 {"id":7375083358042639761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bde3c208-9ab8-43b0-83b6-e85ccfc53e9b {"id":"bde3c208-9ab8-43b0-83b6-e85ccfc53e9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9be2d0a bde3c208-9ab8-43b0-83b6-e85ccfc53e9b /tidb/cdc/default/default/upstream/7375083358042639761 {"id":7375083358042639761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bde3c208-9ab8-43b0-83b6-e85ccfc53e9b {"id":"bde3c208-9ab8-43b0-83b6-e85ccfc53e9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9be2d0a bde3c208-9ab8-43b0-83b6-e85ccfc53e9b /tidb/cdc/default/default/upstream/7375083358042639761 {"id":7375083358042639761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7375083358042639761,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-05-31T16:53:41.610403115+08:00","start_ts":450139421800398852,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139421800398852,"checkpoint_ts":450139421800398852,"checkpoint_time":"2024-05-31 16:53:41.492"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7375083358042639761, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450139421800398852, "checkpoint_time": "2024-05-31 16:53:41.492", "error": null }' + echo '{ "upstream_id": 7375083358042639761, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450139421800398852, "checkpoint_time": "2024-05-31 16:53:41.492", "error": null }' { "upstream_id": 7375083358042639761, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 450139421800398852, "checkpoint_time": "2024-05-31 16:53:41.492", "error": null } ++ echo '{' '"upstream_id":' 7375083358042639761, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 450139421800398852, '"checkpoint_time":' '"2024-05-31' '16:53:41.492",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375083358042639761, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 450139421800398852, '"checkpoint_time":' '"2024-05-31' '16:53:41.492",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/cenkalti/backoff/v4 v4.0.2 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading k8s.io/api v0.24.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/go-playground/locales v0.14.1 go: downloading k8s.io/apimachinery v0.24.0 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading github.com/go-logr/logr v1.2.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:53:48 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 3.93 secs (607871489 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.t2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:53:53 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2426024262.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "updateproportion"="0" "insertproportion"="0" "readproportion"="0" "scanproportion"="0" "operationcount"="0" "mysql.user"="root" "dotransactions"="false" "recordcount"="5000" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "mysql.db"="big_txn" "mysql.port"="4000" "mysql.host"="127.0.0.1" "fieldcount"="100" "threadcount"="1" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:55.314 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:55.314 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c27700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:1071, start at 2024-05-31 16:53:55.315825888 +0800 CST m=+4.354977536 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:55:55.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.292 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.292 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:53:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {ClusterID:default ID:default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 Info:{"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} Status:0xc000ad4df8 TaskPositions:map[c02449d3-d271-4abd-95a8-1039153dbb29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 89, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 82, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 94, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 84, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 87, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 88, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 91, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c02449d3-d271-4abd-95a8-1039153dbb29 {"id":"c02449d3-d271-4abd-95a8-1039153dbb29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9269091 c02449d3-d271-4abd-95a8-1039153dbb29 /tidb/cdc/default/default/changefeed/info/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} /tidb/cdc/default/default/changefeed/status/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":450139425375256583,"min-table-barrier-ts":450139425375256583,"admin-job-type":0} /tidb/cdc/default/default/task/position/c02449d3-d271-4abd-95a8-1039153dbb29/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {ClusterID:default ID:default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 Info:{"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} Status:0xc000ad4df8 TaskPositions:map[c02449d3-d271-4abd-95a8-1039153dbb29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 89, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 82, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 94, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 84, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 87, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 88, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 91, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c02449d3-d271-4abd-95a8-1039153dbb29 {"id":"c02449d3-d271-4abd-95a8-1039153dbb29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9269091 c02449d3-d271-4abd-95a8-1039153dbb29 /tidb/cdc/default/default/changefeed/info/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} /tidb/cdc/default/default/changefeed/status/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":450139425375256583,"min-table-barrier-ts":450139425375256583,"admin-job-type":0} /tidb/cdc/default/default/task/position/c02449d3-d271-4abd-95a8-1039153dbb29/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {ClusterID:default ID:default/53c3a6ee-30e8-4a9d-82bc-0675363cc529 Info:{"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} Status:0xc000ad4df8 TaskPositions:map[c02449d3-d271-4abd-95a8-1039153dbb29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 89, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 82, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 94, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 84, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 87, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 88, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating tableID: 91, resolvedTs: 450139425375256584, checkpointTs: 450139425375256583, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c02449d3-d271-4abd-95a8-1039153dbb29 {"id":"c02449d3-d271-4abd-95a8-1039153dbb29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdd9269091 c02449d3-d271-4abd-95a8-1039153dbb29 /tidb/cdc/default/default/changefeed/info/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"upstream-id":7375083188068745210,"namespace":"default","changefeed-id":"53c3a6ee-30e8-4a9d-82bc-0675363cc529","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:52:04.227087657+08:00","start-ts":450139396264165381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-at+ grep -q 'etcd info' omicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139396290117639} /tidb/cdc/default/default/changefeed/status/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":450139425375256583,"min-table-barrier-ts":450139425375256583,"admin-job-type":0} /tidb/cdc/default/default/task/position/c02449d3-d271-4abd-95a8-1039153dbb29/53c3a6ee-30e8-4a9d-82bc-0675363cc529 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375083188068745210 {"id":7375083188068745210,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 INSERT - Takes(s): 10.0, Count: 213, OPS: 21.3, Avg(us): 45525, Min(us): 1818, Max(us): 1592790, 95th(us): 202000, 99th(us): 913000 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 9.57 secs (249474547 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:881, start at 2024-05-31 16:51:38.49148584 +0800 CST m=+4.445702284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3763cd680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:988, start at 2024-05-31 16:51:38.492378418 +0800 CST m=+4.387981230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:54:38.570 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:51:38.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:41:38.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:54:05 CST 2024] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.2494124943.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:54:08 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92245d25-e5f7-4d82-b3a5-560ef64b3d1c {"id":"92245d25-e5f7-4d82-b3a5-560ef64b3d1c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddb0d878e 92245d25-e5f7-4d82-b3a5-560ef64b3d1c /tidb/cdc/default/default/upstream/7375083725428797009 {"id":7375083725428797009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92245d25-e5f7-4d82-b3a5-560ef64b3d1c {"id":"92245d25-e5f7-4d82-b3a5-560ef64b3d1c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddb0d878e 92245d25-e5f7-4d82-b3a5-560ef64b3d1c /tidb/cdc/default/default/upstream/7375083725428797009 {"id":7375083725428797009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92245d25-e5f7-4d82-b3a5-560ef64b3d1c {"id":"92245d25-e5f7-4d82-b3a5-560ef64b3d1c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddb0d878e 92245d25-e5f7-4d82-b3a5-560ef64b3d1c /tidb/cdc/default/default/upstream/7375083725428797009 {"id":7375083725428797009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.25015.out cli changefeed create --start-ts=450139428105224193 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: 849a8b2a-1a77-4707-9fa5-e3b412fa12c5 Info: {"upstream_id":7375083725428797009,"namespace":"default","id":"849a8b2a-1a77-4707-9fa5-e3b412fa12c5","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:54:09.006522134+08:00","start_ts":450139428105224193,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139428105224193,"checkpoint_ts":450139428105224193,"checkpoint_time":"2024-05-31 16:54:05.543"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:10.446 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:10.446 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d0dc00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:1026, start at 2024-05-31 16:54:10.072189716 +0800 CST m=+19.878688242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:10.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log INSERT - Takes(s): 20.0, Count: 882, OPS: 44.0, Avg(us): 22390, Min(us): 1818, Max(us): 1592790, 95th(us): 99000, 99th(us): 411000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 6.19 secs (385920789 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading cloud.google.com/go/compute v1.13.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 check_redo_resolved_ts 53c3a6ee-30e8-4a9d-82bc-0675363cc529 450139431352139781 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/53c3a6ee-30e8-4a9d-82bc-0675363cc529/meta global resolved ts 450139425676460056 not forward to 450139431352139781 run task failed 1-th time, retry later go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d85980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:27080, start at 2024-05-31 16:54:17.739814169 +0800 CST m=+7.569941726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check_redo_resolved_ts 53c3a6ee-30e8-4a9d-82bc-0675363cc529 450139431352139781 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/53c3a6ee-30e8-4a9d-82bc-0675363cc529/meta global resolved ts 450139425676460056 not forward to 450139431352139781 run task failed 2-th time, retry later Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 5.23 secs (456506286 bytes/sec) [Pipeline] { [Pipeline] cache Starting CDC state checker... INSERT - Takes(s): 30.0, Count: 2393, OPS: 79.8, Avg(us): 12242, Min(us): 1813, Max(us): 1592790, 95th(us): 68000, 99th(us): 197000 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_redo_resolved_ts 53c3a6ee-30e8-4a9d-82bc-0675363cc529 450139431352139781 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/53c3a6ee-30e8-4a9d-82bc-0675363cc529/meta run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/31 16:54:24.864 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 16:54:24.865 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 16:54:24.865 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139425741996033] [resolvedTs=450139432764571650] [2024/05/31 16:54:24.870 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 16:54:24.870 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 16:54:24.871 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=6.124146ms] [2024/05/31 16:54:24.875 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 16:54:24.877 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 16:54:24.879 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 16:54:24.879 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 16:54:24.879 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 16:54:24.882 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 16:54:24.885 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 16:54:25.475 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=610.558706ms] [2024/05/31 16:54:25.476 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":450139425728888839,\"CommitTs\":450139425741996033,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_file\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/31 16:54:26.539 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139432764571650] [2024/05/31 16:54:26.540 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139432764571650] [2024/05/31 16:54:26.640 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139432764571650] [2024/05/31 16:54:26.640 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139432764571650] [2024/05/31 16:54:27.142 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450139432764571650] [2024/05/31 16:54:27.142 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450139432764571650] [2024/05/31 16:54:27.243 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=450139432764571650] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 16:54:27.243 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] Apply redo log successfully check diff successfully [Fri May 31 16:54:27 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 5.55 secs (429892121 bytes/sec) [Pipeline] { [Pipeline] cache INSERT - Takes(s): 40.0, Count: 2620, OPS: 65.5, Avg(us): 14972, Min(us): 1813, Max(us): 1592790, 95th(us): 83000, 99th(us): 205000 start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release INSERT - Takes(s): 50.0, Count: 3222, OPS: 64.5, Avg(us): 15231, Min(us): 1797, Max(us): 1592790, 95th(us): 85000, 99th(us): 239000 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get status pass test: health [2024/05/31 16:54:36.304 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] pass test: changefeed apis [2024/05/31 16:54:39.954 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] pass test: delete changefeed apis pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit ++ date + echo '[Fri May 31 16:54:43 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Fri May 31 16:54:43 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d85980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:27080, start at 2024-05-31 16:54:17.739814169 +0800 CST m=+7.569941726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 13.98 secs (170781356 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 60.0, Count: 4784, OPS: 79.7, Avg(us): 12245, Min(us): 1765, Max(us): 1592790, 95th(us): 72000, 99th(us): 194000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f830c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25618, start at 2024-05-31 16:54:50.329442643 +0800 CST m=+5.464983662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.339 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Run finished, takes 1m1.688934013s INSERT - Takes(s): 61.7, Count: 5000, OPS: 81.1, Avg(us): 12041, Min(us): 1765, Max(us): 1592790, 95th(us): 70000, 99th(us): 186000 table big_txn.usertable exists Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sequence_test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d8090001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:26983, start at 2024-05-31 16:54:17.429643136 +0800 CST m=+7.358779909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.440 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d85980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:27080, start at 2024-05-31 16:54:17.739814169 +0800 CST m=+7.569941726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:17.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:17.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:17.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:55:05 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.3120431206.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 1146 (42S02) at line 1: Table 'sequence_test.seq0' doesn't exist table sequence_test.mark_table not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:55:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c20fac-c34c-492b-b4ca-d6dfd481190e {"id":"62c20fac-c34c-492b-b4ca-d6dfd481190e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddbf2ce88 62c20fac-c34c-492b-b4ca-d6dfd481190e /tidb/cdc/default/default/upstream/7375084383926109705 {"id":7375084383926109705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c20fac-c34c-492b-b4ca-d6dfd481190e {"id":"62c20fac-c34c-492b-b4ca-d6dfd481190e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddbf2ce88 62c20fac-c34c-492b-b4ca-d6dfd481190e /tidb/cdc/default/default/upstream/7375084383926109705 {"id":7375084383926109705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62c20fac-c34c-492b-b4ca-d6dfd481190e {"id":"62c20fac-c34c-492b-b4ca-d6dfd481190e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddbf2ce88 62c20fac-c34c-492b-b4ca-d6dfd481190e /tidb/cdc/default/default/upstream/7375084383926109705 {"id":7375084383926109705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sequence_test.mark_table exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:55:10 CST 2024] <<<<<< run test case sequence success! >>>>>> Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a6680012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30533, start at 2024-05-31 16:55:08.97482229 +0800 CST m=+5.802062722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 22.30 secs (107092785 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G15 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group15: storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=76e1dde5-8aa6-40f8-890a-44fbcc98a63a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-vpkwz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-vpkwz pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri May 31 16:55:13 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri May 31 16:55:13 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G2 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group2: capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=37c551b8-7ce4-4b73-a7ae-f19dbafe6aa0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G2 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-tl5d3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-tl5d3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Starting CDC state checker... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G11 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group11: multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=c628cd82-865c-4c4d-aa54-e44869df8988 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-234hg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-234hg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G9 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group9: kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=cfcff480-8037-48a6-bf1f-5858b3c64b5c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G9 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-ws0dr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-ws0dr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:55:14 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> [Pipeline] sh Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G7 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group7: default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=9dd62179-9353-4a94-a00f-331313810051 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G7 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-s18hd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-s18hd pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G12 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group12: owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=c38d37a3-b24a-4947-a547-28c778bc1441 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-dnm30 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-dnm30 pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G10 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group10: many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=03aea210-f74d-49e8-8a67-57a2086cf0dd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-39q36 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-39q36 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:55:17 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:55:20 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... [Pipeline] stage [Pipeline] { (Test) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:27.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:27.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c4b00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:1033, start at 2024-05-31 16:55:27.322282242 +0800 CST m=+4.358490267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:27.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f830c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25618, start at 2024-05-31 16:54:50.329442643 +0800 CST m=+5.464983662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.339 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc) 2387871232 bytes in 4.60 secs (519509235 bytes/sec) [Pipeline] { Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading cloud.google.com/go v0.105.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:30.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:30.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f50c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:1046, start at 2024-05-31 16:55:30.391958558 +0800 CST m=+4.365977903 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:30.398 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.371 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.371 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:31.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:31.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f9b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1091, start at 2024-05-31 16:55:30.701256586 +0800 CST m=+4.315435546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:30.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 [Pipeline] sh go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G6 Total number of valid directories: 100 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_dup_error_restart changefeed_error Number of directories in group3: 7 changefeed_fast_fail changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group6: ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Sink type: mysql PROW_JOB_ID=837741d8-88fe-4a82-8a29-b3a48c8902f1 JENKINS_NODE_COOKIE=76f81e64-e714-4dbd-8561-2ba04e1114a2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1796461053186412546","prowjobid":"837741d8-88fe-4a82-8a29-b3a48c8902f1","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","base_link":"https://github.com/pingcap/tiflow/commit/6277d9a8c9e9b43613b6d78e2af8aca1fde9593f","pulls":[{"number":10946,"author":"ti-chi-bot","sha":"1b2b17ca406d5b7d993b0a238f16791becd777cb","title":"mounter(ticdc): timezone fill default value should also consider tz. (#10932)","link":"https://github.com/pingcap/tiflow/pull/10946","commit_link":"https://github.com/pingcap/tiflow/pull/10946/commits/1b2b17ca406d5b7d993b0a238f16791becd777cb","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#253 TEST_GROUP=G6 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796461053186412546 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/253/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-xq76s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 pingcap_tiflow_release-6_5_pull_cdc_integration_test_253-xq76s GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=253 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720f3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:990, start at 2024-05-31 16:55:32.09041787 +0800 CST m=+8.526471729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.098 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.098 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a6680012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30533, start at 2024-05-31 16:55:08.97482229 +0800 CST m=+5.802062722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.538 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 Starting CDC state checker... go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37721a540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26207, start at 2024-05-31 16:55:32.795727308 +0800 CST m=+4.411390303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226dc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:1025, start at 2024-05-31 16:55:33.56303525 +0800 CST m=+6.829368922 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.572 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.538 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:1029, start at 2024-05-31 16:55:33.572164593 +0800 CST m=+6.001765790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:33.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go v0.105.0 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 Starting CDC state checker... go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading gopkg.in/yaml.v3 v3.0.1 start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 Starting CDC state checker... go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/zap v1.23.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/coreos/go-semver v0.3.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/fatih/color v1.13.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:38.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:38.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37727c340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1233, start at 2024-05-31 16:55:39.025132647 +0800 CST m=+5.729309318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:39.033 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:39.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:39.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 Starting CDC state checker... go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading golang.org/x/sync v0.5.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading golang.org/x/text v0.14.0 start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading cloud.google.com/go v0.105.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:44.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:44.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d1600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:1037, start at 2024-05-31 16:55:44.475243752 +0800 CST m=+4.261378699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:44.482 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 Verifying downstream PD is started... Starting CDC state checker... go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading golang.org/x/time v0.2.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f85180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25532, start at 2024-05-31 16:54:50.477959657 +0800 CST m=+5.691524376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376f830c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:25618, start at 2024-05-31 16:54:50.329442643 +0800 CST m=+5.464983662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:50.339 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:50.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:50.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:55:49 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.2970829710.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b279c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:1045, start at 2024-05-31 16:53:38.957232437 +0800 CST m=+4.670104670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.969 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.969 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:55:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a351b0c-f80a-4870-8537-b2d104ede957 {"id":"7a351b0c-f80a-4870-8537-b2d104ede957","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca2218d 7a351b0c-f80a-4870-8537-b2d104ede957 /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a351b0c-f80a-4870-8537-b2d104ede957 {"id":"7a351b0c-f80a-4870-8537-b2d104ede957","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca2218d 7a351b0c-f80a-4870-8537-b2d104ede957 /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a351b0c-f80a-4870-8537-b2d104ede957 {"id":"7a351b0c-f80a-4870-8537-b2d104ede957","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca2218d 7a351b0c-f80a-4870-8537-b2d104ede957 /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a0840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30449, start at 2024-05-31 16:55:08.625544309 +0800 CST m=+5.516195902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3770a6680012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:30533, start at 2024-05-31 16:55:08.97482229 +0800 CST m=+5.802062722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:08.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:08.954 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:08.954 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:55:53 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3462534627.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:55:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8118f87-225c-4bb5-b453-69734866e37c {"id":"d8118f87-225c-4bb5-b453-69734866e37c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcaf7c90 d8118f87-225c-4bb5-b453-69734866e37c /tidb/cdc/default/default/upstream/7375084615704992503 {"id":7375084615704992503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8118f87-225c-4bb5-b453-69734866e37c {"id":"d8118f87-225c-4bb5-b453-69734866e37c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcaf7c90 d8118f87-225c-4bb5-b453-69734866e37c /tidb/cdc/default/default/upstream/7375084615704992503 {"id":7375084615704992503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8118f87-225c-4bb5-b453-69734866e37c {"id":"d8118f87-225c-4bb5-b453-69734866e37c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcaf7c90 d8118f87-225c-4bb5-b453-69734866e37c /tidb/cdc/default/default/upstream/7375084615704992503 {"id":7375084615704992503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: fa8e7465-e091-41af-b339-57a955c22e61 Info: {"upstream_id":7375084615704992503,"namespace":"default","id":"fa8e7465-e091-41af-b339-57a955c22e61","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:55:56.847712269+08:00","start_ts":450139456429883396,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139456429883396,"checkpoint_ts":450139456429883396,"checkpoint_time":"2024-05-31 16:55:53.593"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37721a540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26207, start at 2024-05-31 16:55:32.795727308 +0800 CST m=+4.411390303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:55:57.316 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc000a496c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] [2024/05/31 16:55:57.320 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0008096c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"] start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table big_txn.usertable1 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c27700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:1071, start at 2024-05-31 16:53:55.315825888 +0800 CST m=+4.354977536 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.394 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.292 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.292 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b26dc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:944, start at 2024-05-31 16:53:38.894134971 +0800 CST m=+4.691384670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376b279c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:1045, start at 2024-05-31 16:53:38.957232437 +0800 CST m=+4.670104670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:39.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:38.969 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:38.969 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:56:11 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sinkv2/eventsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2604326045.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:56:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.26114.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 74258dd0-b372-45ca-a070-da92a661942b Info: {"upstream_id":7375084235286715637,"namespace":"default","id":"74258dd0-b372-45ca-a070-da92a661942b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:56:14.757382283+08:00","start_ts":450139461941723141,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139461941723141,"checkpoint_ts":450139461941723141,"checkpoint_time":"2024-05-31 16:56:14.619"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 1-th time, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377218140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26109, start at 2024-05-31 16:55:32.636265747 +0800 CST m=+4.336213985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37721a540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:26207, start at 2024-05-31 16:55:32.795727308 +0800 CST m=+4.411390303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:57:32.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30324.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139462546751489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139462546751489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c2780000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:973, start at 2024-05-31 16:53:55.307937633 +0800 CST m=+4.417313826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376c27700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:1071, start at 2024-05-31 16:53:55.315825888 +0800 CST m=+4.354977536 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:56:55.394 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:53:55.292 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:43:55.292 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:56:27 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.2476024762.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:56:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 {"id":"4b837c1a-337f-4a0c-bb78-3fd24b81b8b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd34db90 4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 /tidb/cdc/default/default/upstream/7375084314195619056 {"id":7375084314195619056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 {"id":"4b837c1a-337f-4a0c-bb78-3fd24b81b8b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd34db90 4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 /tidb/cdc/default/default/upstream/7375084314195619056 {"id":7375084314195619056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 {"id":"4b837c1a-337f-4a0c-bb78-3fd24b81b8b9","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd34db90 4b837c1a-337f-4a0c-bb78-3fd24b81b8b9 /tidb/cdc/default/default/upstream/7375084314195619056 {"id":7375084314195619056,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: dde14c05-dd80-4a06-bff9-9324bbca8391 Info: {"upstream_id":7375084314195619056,"namespace":"default","id":"dde14c05-dd80-4a06-bff9-9324bbca8391","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:56:30.443943863+08:00","start_ts":450139465235562497,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139465235562497,"checkpoint_ts":450139465235562497,"checkpoint_time":"2024-05-31 16:56:27.184"} Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d0dc00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:1026, start at 2024-05-31 16:54:10.072189716 +0800 CST m=+19.878688242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.110 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ***************** properties ***************** "scanproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="100" "readallfields"="true" "mysql.user"="root" "mysql.db"="consistent_replicate_storage_file_large_value" "insertproportion"="0" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "threadcount"="10" "fieldcount"="1000" "fieldlengthdistribution"="constant" "fieldlength"="15000" "requestdistribution"="uniform" ********************************************** Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table autorandom_test.table_a not exists for 1-th check, retry later table autorandom_test.table_a exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d13a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:930, start at 2024-05-31 16:54:10.438062551 +0800 CST m=+20.316759692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f376d0dc00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:1026, start at 2024-05-31 16:54:10.072189716 +0800 CST m=+19.878688242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:10.110 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:54:10.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:44:10.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.24275.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:56:55 CST 2024] <<<<<< run test case autorandom success! >>>>>> + set +x + tso='450139472712433665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139472712433665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:56:56 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.2431224314.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 INSERT_ERROR - Takes(s): 8.5, Count: 63, OPS: 7.5, Avg(us): 898236, Min(us): 442882, Max(us): 1744959, 95th(us): 0, 99th(us): 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:56:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9da5aad1-d761-44cc-b5b4-bc915b071ae7 {"id":"9da5aad1-d761-44cc-b5b4-bc915b071ae7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddda15490 9da5aad1-d761-44cc-b5b4-bc915b071ae7 /tidb/cdc/default/default/upstream/7375084311612146622 {"id":7375084311612146622,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9da5aad1-d761-44cc-b5b4-bc915b071ae7 {"id":"9da5aad1-d761-44cc-b5b4-bc915b071ae7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddda15490 9da5aad1-d761-44cc-b5b4-bc915b071ae7 /tidb/cdc/default/default/upstream/7375084311612146622 {"id":7375084311612146622,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9da5aad1-d761-44cc-b5b4-bc915b071ae7 {"id":"9da5aad1-d761-44cc-b5b4-bc915b071ae7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddda15490 9da5aad1-d761-44cc-b5b4-bc915b071ae7 /tidb/cdc/default/default/upstream/7375084311612146622 {"id":7375084311612146622,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.24379.out cli changefeed create --start-ts=450139472712433665 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: afa6cf14-b583-45c4-8cf4-09eca6722e27 Info: {"upstream_id":7375084311612146622,"namespace":"default","id":"afa6cf14-b583-45c4-8cf4-09eca6722e27","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:56:59.761331164+08:00","start_ts":450139472712433665,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139472712433665,"checkpoint_ts":450139472712433665,"checkpoint_time":"2024-05-31 16:56:55.706"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Run finished, takes 13.32119117s INSERT_ERROR - Takes(s): 11.8, Count: 100, OPS: 8.5, Avg(us): 757833, Min(us): 250091, Max(us): 1744959, 95th(us): 0, 99th(us): 0 table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later table processor_resolved_ts_fallback.t1 exists [Fri May 31 16:57:12 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2630726309.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:57:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/74258dd0-b372-45ca-a070-da92a661942b {ClusterID:default ID:default/74258dd0-b372-45ca-a070-da92a661942b Info:{"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} Status:0xc00392c750 TaskPositions:map[688b5dac-4902-4cfa-bccb-277f30062d55:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c8bb043a-bd4d-4292-a359-6916da6e22b2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c8bb043a-bd4d-4292-a359-6916da6e22b2 {"id":"c8bb043a-bd4d-4292-a359-6916da6e22b2","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf533c0 c8bb043a-bd4d-4292-a359-6916da6e22b2 /tidb/cdc/default/default/changefeed/info/74258dd0-b372-45ca-a070-da92a661942b {"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} /tidb/cdc/default/default/changefeed/status/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":450139477670363143,"min-table-barrier-ts":450139477670363143,"admin-job-type":0} /tidb/cdc/default/default/task/position/688b5dac-4902-4cfa-bccb-277f30062d55/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c8bb043a-bd4d-4292-a359-6916da6e22b2/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/74258dd0-b372-45ca-a070-da92a661942b {ClusterID:default ID:default/74258dd0-b372-45ca-a070-da92a661942b Info:{"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} Status:0xc00392c750 TaskPositions:map[688b5dac-4902-4cfa-bccb-277f30062d55:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c8bb043a-bd4d-4292-a359-6916da6e22b2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c8bb043a-bd4d-4292-a359-6916da6e22b2 {"id":"c8bb043a-bd4d-4292-a359-6916da6e22b2","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf533c0 c8bb043a-bd4d-4292-a359-6916da6e22b2 /tidb/cdc/default/default/changefeed/info/74258dd0-b372-45ca-a070-da92a661942b {"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use+ grep -q 'failed to get info:' -file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} /tidb/cdc/default/default/changefeed/status/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":450139477670363143,"min-table-barrier-ts":450139477670363143,"admin-job-type":0} /tidb/cdc/default/default/task/position/688b5dac-4902-4cfa-bccb-277f30062d55/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c8bb043a-bd4d-4292-a359-6916da6e22b2/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/74258dd0-b372-45ca-a070-da92a661942b {ClusterID:default ID:default/74258dd0-b372-45ca-a070-da92a661942b Info:{"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} Status:0xc00392c750 TaskPositions:map[688b5dac-4902-4cfa-bccb-277f30062d55:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c8bb043a-bd4d-4292-a359-6916da6e22b2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688b5dac-4902-4cfa-bccb-277f30062d55 {"id":"688b5dac-4902-4cfa-bccb-277f30062d55","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c8bb043a-bd4d-4292-a359-6916da6e22b2 {"id":"c8bb043a-bd4d-4292-a359-6916da6e22b2","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf5328e 688b5dac-4902-4cfa-bccb-277f30062d55 /tidb/cdc/default/__cdc_meta__/owner/22318fcddcf533c0 c8bb043a-bd4d-4292-a359-6916da6e22b2 /tidb/cdc/default/default/changefeed/info/74258dd0-b372-45ca-a070-da92a661942b {"upstream-id":7375084235286715637,"namespace":"default","changefeed-id":"74258dd0-b372-45ca-a070-da92a661942b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:56:14.757382283+08:00","start-ts":450139461941723141,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139461968199686} /tidb/cdc/default/default/changefeed/status/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":450139477670363143,"min-table-barrier-ts":450139477670363143,"admin-job-type":0} /tidb/cdc/default/default/task/position/688b5dac-4902-4cfa-bccb-277f30062d55/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c8bb043a-bd4d-4292-a359-6916da6e22b2/74258dd0-b372-45ca-a070-da92a661942b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084235286715637 {"id":7375084235286715637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later table consistent_replicate_storage_file_large_value.t2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:18 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.3005930061.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788b680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35918, start at 2024-05-31 16:57:18.339429929 +0800 CST m=+4.989145447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 26056 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:57:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/38c44990-04d4-4279-af63-69514bc7227e {ClusterID:default ID:default/38c44990-04d4-4279-af63-69514bc7227e Info:{"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} Status:0xc00196f440 TaskPositions:map[3f9ac2bd-db45-4a66-be8f-754f564211ce:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 84, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 91, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 94, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 87, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 82, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 92, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 88, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f9ac2bd-db45-4a66-be8f-754f564211ce {"id":"3f9ac2bd-db45-4a66-be8f-754f564211ce","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca22359 3f9ac2bd-db45-4a66-be8f-754f564211ce /tidb/cdc/default/default/changefeed/info/38c44990-04d4-4279-af63-69514bc7227e {"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} /tidb/cdc/default/default/changefeed/status/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":450139479166681091,"min-table-barrier-ts":450139479428825091,"admin-job-type":0} /tidb/cdc/default/default/task/position/3f9ac2bd-db45-4a66-be8f-754f564211ce/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/38c44990-04d4-4279-af63-69514bc7227e {ClusterID:default ID:default/38c44990-04d4-4279-af63-69514bc7227e Info:{"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} Status:0xc00196f440 TaskPositions:map[3f9ac2bd-db45-4a66-be8f-754f564211ce:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 84, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 91, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 94, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 87, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 82, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 92, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 88, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f9ac2bd-db45-4a66-be8f-754f564211ce {"id":"3f9ac2bd-db45-4a66-be8f-754f564211ce","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca22359 3f9ac2bd-db45-4a66-be8f-754f564211ce /tidb/cdc/default/default/changefeed/info/38c44990-04d4-4279-af63-69514bc7227e {"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} /tidb/cdc/default/default/changefeed/status/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":450139479166681091,"min-table-barrier-ts":450139479428825091,"admin-job-type":0} /tidb/cdc/default/default/task/position/3f9ac2bd-db45-4a66-be8f-754f564211ce/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/38c44990-04d4-4279-af63-69514bc7227e {ClusterID:default ID:default/38c44990-04d4-4279-af63-69514bc7227e Info:{"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} Status:0xc00196f440 TaskPositions:map[3f9ac2bd-db45-4a66-be8f-754f564211ce:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 89, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 84, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 91, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 94, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 87, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 82, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 92, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating tableID: 88, resolvedTs: 450139479166681092, checkpointTs: 450139479166681091, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f9ac2bd-db45-4a66-be8f-754f564211ce {"id":"3f9ac2bd-db45-4a66-be8f-754f564211ce","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddca22359 3f9ac2bd-db45-4a66-be8f-754f564211ce /tidb/cdc/default/default/changefeed/info/38c44990-04d4-4279-af63-69514bc7227e {"upstream-id":7375084544156457991,"namespace":"default","changefeed-id":"38c44990-04d4-4279-af63-69514bc7227e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:55:52.332290732+08:00","start-ts":450139456071794696,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139456098009090} /tidb/cdc/default/default/changefeed/status/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":450139479166681091,"min-table-barrier-ts":450139479428825091,"admin-job-type":0} /tidb/cdc/default/default/task/position/3f9ac2bd-db45-4a66-be8f-754f564211ce/38c44990-04d4-4279-af63-69514bc7227e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084544156457991 {"id":7375084544156457991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Fri May 31 16:57:13 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.3045530457.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:57:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 {"id":"9bbb59cc-d4e7-40fd-91ad-0124a4c70c23","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd0a1011 9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 /tidb/cdc/default/default/upstream/7375084734759121070 {"id":7375084734759121070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 {"id":"9bbb59cc-d4e7-40fd-91ad-0124a4c70c23","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd0a1011 9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 /tidb/cdc/default/default/upstream/7375084734759121070 {"id":7375084734759121070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 {"id":"9bbb59cc-d4e7-40fd-91ad-0124a4c70c23","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddd0a1011 9bbb59cc-d4e7-40fd-91ad-0124a4c70c23 /tidb/cdc/default/default/upstream/7375084734759121070 {"id":7375084734759121070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30523.out cli changefeed create --start-ts=450139462546751489 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: e955e807-44b5-4cb1-ba75-5c4b37a2a07b Info: {"upstream_id":7375084734759121070,"namespace":"default","id":"e955e807-44b5-4cb1-ba75-5c4b37a2a07b","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:57:17.32944208+08:00","start_ts":450139462546751489,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139462546751489,"checkpoint_ts":450139462546751489,"checkpoint_time":"2024-05-31 16:56:16.927"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:24 CST 2024] <<<<<< run test case simple success! >>>>>> cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later Verifying downstream PD is started... table clustered_index_test.t0 exists table clustered_index_test.t1 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table clustered_index_test.t1 exists table clustered_index_test.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c4b00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:1033, start at 2024-05-31 16:55:27.322282242 +0800 CST m=+4.358490267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:31.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:31.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f9b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1091, start at 2024-05-31 16:55:30.701256586 +0800 CST m=+4.315435546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:30.728 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff successfully check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:35 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720f3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:990, start at 2024-05-31 16:55:32.09041787 +0800 CST m=+8.526471729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.098 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.098 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... check diff failed 7-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 38c44990-04d4-4279-af63-69514bc7227e 450139485051813891 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/38c44990-04d4-4279-af63-69514bc7227e/meta global resolved ts 450139484409561094 not forward to 450139485051813891 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:1029, start at 2024-05-31 16:55:33.572164593 +0800 CST m=+6.001765790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check_redo_resolved_ts 38c44990-04d4-4279-af63-69514bc7227e 450139485051813891 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/38c44990-04d4-4279-af63-69514bc7227e/meta global resolved ts 450139484933849095 not forward to 450139485051813891 run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226dc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:1025, start at 2024-05-31 16:55:33.56303525 +0800 CST m=+6.829368922 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 8-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d1600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:1037, start at 2024-05-31 16:55:44.475243752 +0800 CST m=+4.261378699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... check diff failed 9-th time, retry later Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a42100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31621, start at 2024-05-31 16:57:46.397159919 +0800 CST m=+4.487836998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:38.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:38.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37727c340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1233, start at 2024-05-31 16:55:39.025132647 +0800 CST m=+5.729309318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:39.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:39.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:39.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check_redo_resolved_ts 38c44990-04d4-4279-af63-69514bc7227e 450139485051813891 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/38c44990-04d4-4279-af63-69514bc7227e/meta run task successfully check diff failed 10-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 16:57:49.677 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 16:57:49.677 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 16:57:49.677 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139485995270146] [resolvedTs=450139486519558145] [2024/05/31 16:57:49.678 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=243.33µs] [2024/05/31 16:57:49.678 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=259.844µs] [2024/05/31 16:57:49.678 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 16:57:49.678 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 16:57:49.683 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 16:57:49.685 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 16:57:49.686 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 16:57:49.687 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 16:57:49.687 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 16:57:49.687 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 16:57:49.691 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 16:57:49.693 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 16:57:49.693 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=450139486519558145] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 16:57:49.693 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] Apply redo log successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c5880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:955, start at 2024-05-31 16:55:27.337086347 +0800 CST m=+4.467311968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771c4b00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:1033, start at 2024-05-31 16:55:27.322282242 +0800 CST m=+4.358490267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:27.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:27.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:27.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff successfully [Fri May 31 16:57:50 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788b680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35918, start at 2024-05-31 16:57:18.339429929 +0800 CST m=+4.989145447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 11-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:31.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771fe700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1028, start at 2024-05-31 16:55:31.012476534 +0800 CST m=+4.696750459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:31.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:31.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:31.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f9b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:1091, start at 2024-05-31 16:55:30.701256586 +0800 CST m=+4.315435546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:30.728 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.17914.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139487722012674 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139487722012674 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:57:53 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.1795617958.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... check_changefeed_state http://127.0.0.1:2379 38560039-a19d-4b43-95cd-3c508bafa558 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=38560039-a19d-4b43-95cd-3c508bafa558 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 38560039-a19d-4b43-95cd-3c508bafa558 -s + info='{ "upstream_id": 7375084383926109705, "namespace": "default", "id": "38560039-a19d-4b43-95cd-3c508bafa558", "state": "finished", "checkpoint_tso": 450139468369231874, "checkpoint_time": "2024-05-31 16:56:39.138", "error": null }' + echo '{ "upstream_id": 7375084383926109705, "namespace": "default", "id": "38560039-a19d-4b43-95cd-3c508bafa558", "state": "finished", "checkpoint_tso": 450139468369231874, "checkpoint_time": "2024-05-31 16:56:39.138", "error": null }' { "upstream_id": 7375084383926109705, "namespace": "default", "id": "38560039-a19d-4b43-95cd-3c508bafa558", "state": "finished", "checkpoint_tso": 450139468369231874, "checkpoint_time": "2024-05-31 16:56:39.138", "error": null } ++ echo '{' '"upstream_id":' 7375084383926109705, '"namespace":' '"default",' '"id":' '"38560039-a19d-4b43-95cd-3c508bafa558",' '"state":' '"finished",' '"checkpoint_tso":' 450139468369231874, '"checkpoint_time":' '"2024-05-31' '16:56:39.138",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7375084383926109705, '"namespace":' '"default",' '"id":' '"38560039-a19d-4b43-95cd-3c508bafa558",' '"state":' '"finished",' '"checkpoint_tso":' 450139468369231874, '"checkpoint_time":' '"2024-05-31' '16:56:39.138",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:39 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 12-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 13-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table generate_column.t not exists for 1-th check, retry later table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:57:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 {"id":"a4cbb8e8-a9f2-441c-970c-17fa9cf2b532","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde873f8e a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 /tidb/cdc/default/default/upstream/7375084725841574308 {"id":7375084725841574308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 {"id":"a4cbb8e8-a9f2-441c-970c-17fa9cf2b532","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde873f8e a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 /tidb/cdc/default/default/upstream/7375084725841574308 {"id":7375084725841574308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 {"id":"a4cbb8e8-a9f2-441c-970c-17fa9cf2b532","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde873f8e a4cbb8e8-a9f2-441c-970c-17fa9cf2b532 /tidb/cdc/default/default/upstream/7375084725841574308 {"id":7375084725841574308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.18015.out cli changefeed create --start-ts=450139487722012674 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 165f9cba-c271-4fea-9263-fb864804bca8 Info: {"upstream_id":7375084725841574308,"namespace":"default","id":"165f9cba-c271-4fea-9263-fb864804bca8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:57:56.680365441+08:00","start_ts":450139487722012674,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139487722012674,"checkpoint_ts":450139487722012674,"checkpoint_time":"2024-05-31 16:57:52.963"} PASS start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 14-th time, retry later table generate_column.t1 exists Verifying downstream PD is started... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:58 CST 2024] <<<<<< run test case generate_column success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aec880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27717, start at 2024-05-31 16:57:57.29196385 +0800 CST m=+4.606333951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 15-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D48714CF93D732 < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 08:58:01 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... check diff failed 16-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1c2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32638, start at 2024-05-31 16:58:00.337138522 +0800 CST m=+4.455001788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720e400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:937, start at 2024-05-31 16:55:32.027459944 +0800 CST m=+8.543700727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.064 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37720f3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:990, start at 2024-05-31 16:55:32.09041787 +0800 CST m=+8.526471729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:32.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:32.098 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:32.098 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:02 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1715617158.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 17-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:964, start at 2024-05-31 16:55:33.529838837 +0800 CST m=+6.059288956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:1029, start at 2024-05-31 16:55:33.572164593 +0800 CST m=+6.001765790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.16984.out cli tso query --pd=http://127.0.0.1:2379 Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37722650000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:977, start at 2024-05-31 16:55:33.545176963 +0800 CST m=+6.881732359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377226dc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:1025, start at 2024-05-31 16:55:33.56303525 +0800 CST m=+6.829368922 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:33.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:33.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:33.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x + tso='450139490616606721 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139490616606721 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:58:04 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.1702117023.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f50c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:1046, start at 2024-05-31 16:55:30.391958558 +0800 CST m=+4.365977903 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.371 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.371 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.16901.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x + tso='450139490823176193 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139490823176193 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:58:05 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.1693716939.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0d15c96-a655-4729-ad77-204c77c9eaac {"id":"d0d15c96-a655-4729-ad77-204c77c9eaac","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddea3db8e d0d15c96-a655-4729-ad77-204c77c9eaac /tidb/cdc/default/default/upstream/7375084707743645133 {"id":7375084707743645133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0d15c96-a655-4729-ad77-204c77c9eaac {"id":"d0d15c96-a655-4729-ad77-204c77c9eaac","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddea3db8e d0d15c96-a655-4729-ad77-204c77c9eaac /tidb/cdc/default/default/upstream/7375084707743645133 {"id":7375084707743645133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0d15c96-a655-4729-ad77-204c77c9eaac {"id":"d0d15c96-a655-4729-ad77-204c77c9eaac","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddea3db8e d0d15c96-a655-4729-ad77-204c77c9eaac /tidb/cdc/default/default/upstream/7375084707743645133 {"id":7375084707743645133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 18-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:940, start at 2024-05-31 16:55:44.441612242 +0800 CST m=+4.277801993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3772d1600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:1037, start at 2024-05-31 16:55:44.475243752 +0800 CST m=+4.261378699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:44.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:44.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:44.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24653.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139491363454977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139491363454977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:58:07 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2469424696.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3b4c81d-1063-4f3b-9511-04a1cf44132a {"id":"f3b4c81d-1063-4f3b-9511-04a1cf44132a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeac2491 f3b4c81d-1063-4f3b-9511-04a1cf44132a /tidb/cdc/default/default/upstream/7375084727723236211 {"id":7375084727723236211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3b4c81d-1063-4f3b-9511-04a1cf44132a {"id":"f3b4c81d-1063-4f3b-9511-04a1cf44132a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeac2491 f3b4c81d-1063-4f3b-9511-04a1cf44132a /tidb/cdc/default/default/upstream/7375084727723236211 {"id":7375084727723236211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3b4c81d-1063-4f3b-9511-04a1cf44132a {"id":"f3b4c81d-1063-4f3b-9511-04a1cf44132a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeac2491 f3b4c81d-1063-4f3b-9511-04a1cf44132a /tidb/cdc/default/default/upstream/7375084727723236211 {"id":7375084727723236211,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.17089.out cli changefeed create --start-ts=450139490616606721 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 487c01c4-6a17-45db-8b32-f09a943858ab Info: {"upstream_id":7375084727723236211,"namespace":"default","id":"487c01c4-6a17-45db-8b32-f09a943858ab","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:58:07.736778624+08:00","start_ts":450139490616606721,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139490616606721,"checkpoint_ts":450139490616606721,"checkpoint_time":"2024-05-31 16:58:04.005"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 19-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:58:07 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:08 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef1d7ab-9d6c-43be-9114-f809a4bd234a {"id":"fef1d7ab-9d6c-43be-9114-f809a4bd234a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeaf358f fef1d7ab-9d6c-43be-9114-f809a4bd234a /tidb/cdc/default/default/upstream/7375084726097667894 {"id":7375084726097667894,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef1d7ab-9d6c-43be-9114-f809a4bd234a {"id":"fef1d7ab-9d6c-43be-9114-f809a4bd234a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeaf358f fef1d7ab-9d6c-43be-9114-f809a4bd234a /tidb/cdc/default/default/upstream/7375084726097667894 {"id":7375084726097667894,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef1d7ab-9d6c-43be-9114-f809a4bd234a {"id":"fef1d7ab-9d6c-43be-9114-f809a4bd234a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeaf358f fef1d7ab-9d6c-43be-9114-f809a4bd234a /tidb/cdc/default/default/upstream/7375084726097667894 {"id":7375084726097667894,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.16991.out cli changefeed create --start-ts=450139490823176193 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:38.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377275d8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1149, start at 2024-05-31 16:55:38.654402829 +0800 CST m=+5.437643091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:38.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:38.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:38.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37727c340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:1233, start at 2024-05-31 16:55:39.025132647 +0800 CST m=+5.729309318 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:58:39.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:39.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:39.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:08 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2455824560.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 76507cb3-2beb-47a4-83cc-0dcdd7d876a6 Info: {"upstream_id":7375084726097667894,"namespace":"default","id":"76507cb3-2beb-47a4-83cc-0dcdd7d876a6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:58:08.549775974+08:00","start_ts":450139490823176193,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139490823176193,"checkpoint_ts":450139490823176193,"checkpoint_time":"2024-05-31 16:58:04.793"} PASS table big_txn.finish_mark_1 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:57:57 CST 2024] <<<<<< run test case big_txn success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:58:06 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788c1c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35812, start at 2024-05-31 16:57:18.354539879 +0800 CST m=+5.067584548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37788b680016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:35918, start at 2024-05-31 16:57:18.339429929 +0800 CST m=+4.989145447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:18.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:18.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:18.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.39970.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:58:09 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> + set +x + tso='450139492064428033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139492064428033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 20-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b1b902-062b-4dc6-96af-ba27d9113080 {"id":"c1b1b902-062b-4dc6-96af-ba27d9113080","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeb58391 c1b1b902-062b-4dc6-96af-ba27d9113080 /tidb/cdc/default/default/upstream/7375084785404379291 {"id":7375084785404379291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b1b902-062b-4dc6-96af-ba27d9113080 {"id":"c1b1b902-062b-4dc6-96af-ba27d9113080","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeb58391 c1b1b902-062b-4dc6-96af-ba27d9113080 /tidb/cdc/default/default/upstream/7375084785404379291 {"id":7375084785404379291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b1b902-062b-4dc6-96af-ba27d9113080 {"id":"c1b1b902-062b-4dc6-96af-ba27d9113080","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddeb58391 c1b1b902-062b-4dc6-96af-ba27d9113080 /tidb/cdc/default/default/upstream/7375084785404379291 {"id":7375084785404379291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24763.out cli changefeed create --start-ts=450139491363454977 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 253d0ba3-efd1-4c29-af6a-e97dafcaf1aa Info: {"upstream_id":7375084785404379291,"namespace":"default","id":"253d0ba3-efd1-4c29-af6a-e97dafcaf1aa","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-31T16:58:10.534550979+08:00","start_ts":450139491363454977,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139491363454977,"checkpoint_ts":450139491363454977,"checkpoint_time":"2024-05-31 16:58:06.854"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a42100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31621, start at 2024-05-31 16:57:46.397159919 +0800 CST m=+4.487836998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07f423bf-3aef-4aa2-99e3-ffc41a8307f6 {"id":"07f423bf-3aef-4aa2-99e3-ffc41a8307f6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf008e 07f423bf-3aef-4aa2-99e3-ffc41a8307f6 /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07f423bf-3aef-4aa2-99e3-ffc41a8307f6 {"id":"07f423bf-3aef-4aa2-99e3-ffc41a8307f6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf008e 07f423bf-3aef-4aa2-99e3-ffc41a8307f6 /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07f423bf-3aef-4aa2-99e3-ffc41a8307f6 {"id":"07f423bf-3aef-4aa2-99e3-ffc41a8307f6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf008e 07f423bf-3aef-4aa2-99e3-ffc41a8307f6 /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info: {"upstream_id":7375084744008070100,"namespace":"default","id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T16:58:11.790585206+08:00","start_ts":450139492629610502,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139492629610502,"checkpoint_ts":450139492629610502,"checkpoint_time":"2024-05-31 16:58:11.684"} Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) check diff failed 21-th time, retry later Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:58:12 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) Installing collected packages: certifi, idna, urllib3, requests Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: requests 2.27.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 check diff failed 22-th time, retry later The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31477, start at 2024-05-31 16:58:13.954763041 +0800 CST m=+4.408712906 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri May 31 16:58:15 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 23-th time, retry later Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aec880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27717, start at 2024-05-31 16:57:57.29196385 +0800 CST m=+4.606333951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:58:20 CST 2024] <<<<<< run test case clustered_index success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1c2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32638, start at 2024-05-31 16:58:00.337138522 +0800 CST m=+4.455001788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f4580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:852, start at 2024-05-31 16:55:30.362411879 +0800 CST m=+4.443238952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3771f50c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:1046, start at 2024-05-31 16:55:30.391958558 +0800 CST m=+4.365977903 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:30.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:55:30.371 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:45:30.371 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:26 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.2111021112.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc6c40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25718, start at 2024-05-31 16:58:27.636479936 +0800 CST m=+4.442157169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.643 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 {"id":"a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf044890 a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 /tidb/cdc/default/default/upstream/7375084713070574261 {"id":7375084713070574261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 {"id":"a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf044890 a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 /tidb/cdc/default/default/upstream/7375084713070574261 {"id":7375084713070574261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 {"id":"a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf044890 a1d7bf6a-ca8b-4ab6-86c0-12d0cc819ae0 /tidb/cdc/default/default/upstream/7375084713070574261 {"id":7375084713070574261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.21157.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: aa336d7f-a4fe-49d0-8356-31c46cce8494 Info: {"upstream_id":7375084713070574261,"namespace":"default","id":"aa336d7f-a4fe-49d0-8356-31c46cce8494","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-31T16:58:29.91476732+08:00","start_ts":450139497372581894,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139497372581894,"checkpoint_ts":450139497372581894,"checkpoint_time":"2024-05-31 16:58:29.777"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a3fc00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31510, start at 2024-05-31 16:57:46.22991931 +0800 CST m=+4.377264067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377a42100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:31621, start at 2024-05-31 16:57:46.397159919 +0800 CST m=+4.487836998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:46.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:46.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:46.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:30 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.3579135793.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cef100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26783, start at 2024-05-31 16:58:30.250283822 +0800 CST m=+4.257961481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.256 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/error.log # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f956c7-5cda-4c22-b616-f0f16629e29c {"id":"e8f956c7-5cda-4c22-b616-f0f16629e29c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf15918e e8f956c7-5cda-4c22-b616-f0f16629e29c /tidb/cdc/default/default/upstream/7375085301742719064 {"id":7375085301742719064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f956c7-5cda-4c22-b616-f0f16629e29c {"id":"e8f956c7-5cda-4c22-b616-f0f16629e29c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf15918e e8f956c7-5cda-4c22-b616-f0f16629e29c /tidb/cdc/default/default/upstream/7375085301742719064 {"id":7375085301742719064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f956c7-5cda-4c22-b616-f0f16629e29c {"id":"e8f956c7-5cda-4c22-b616-f0f16629e29c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf15918e e8f956c7-5cda-4c22-b616-f0f16629e29c /tidb/cdc/default/default/upstream/7375085301742719064 {"id":7375085301742719064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31477, start at 2024-05-31 16:58:13.954763041 +0800 CST m=+4.408712906 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aea2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27618, start at 2024-05-31 16:57:57.165507206 +0800 CST m=+4.550086097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377aec880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:27717, start at 2024-05-31 16:57:57.29196385 +0800 CST m=+4.606333951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-16:59:57.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:57:57.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:47:57.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:39 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.3200332005.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8da921c-5916-4421-83de-2edab7db99d1 {"id":"f8da921c-5916-4421-83de-2edab7db99d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf39308e f8da921c-5916-4421-83de-2edab7db99d1 /tidb/cdc/default/default/upstream/7375085342682781836 {"id":7375085342682781836,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8da921c-5916-4421-83de-2edab7db99d1 {"id":"f8da921c-5916-4421-83de-2edab7db99d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf39308e f8da921c-5916-4421-83de-2edab7db99d1 /tidb/cdc/default/default/upstream/7375085342682781836 {"id":7375085342682781836,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8da921c-5916-4421-83de-2edab7db99d1 {"id":"f8da921c-5916-4421-83de-2edab7db99d1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf39308e f8da921c-5916-4421-83de-2edab7db99d1 /tidb/cdc/default/default/upstream/7375085342682781836 {"id":7375085342682781836,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1a7c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32542, start at 2024-05-31 16:58:00.236821354 +0800 CST m=+4.446934427 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377b1c2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:32638, start at 2024-05-31 16:58:00.337138522 +0800 CST m=+4.455001788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:00.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:44 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.3690336905.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:47 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 {"id":"650d5321-e7f2-4bb7-8ae1-6b7eb03e2341","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf47cd91 650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 /tidb/cdc/default/default/upstream/7375085365741400036 {"id":7375085365741400036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 {"id":"650d5321-e7f2-4bb7-8ae1-6b7eb03e2341","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf47cd91 650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 /tidb/cdc/default/default/upstream/7375085365741400036 {"id":7375085365741400036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 {"id":"650d5321-e7f2-4bb7-8ae1-6b7eb03e2341","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf47cd91 650d5321-e7f2-4bb7-8ae1-6b7eb03e2341 /tidb/cdc/default/default/upstream/7375085365741400036 {"id":7375085365741400036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26719, start at 2024-05-31 16:58:47.224147052 +0800 CST m=+4.273137115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc6c40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25718, start at 2024-05-31 16:58:27.636479936 +0800 CST m=+4.442157169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.643 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:58:50.811 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/31 16:58:50.998 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:58:51.003 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:58:51.448 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 16:58:51.449 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 16:58:51.461 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:58:51.463 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:58:51.887 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:58:51.898 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:58:52.375 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:58:52.380 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:58:52.383 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:58:52.387 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:58:52.883 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 16:58:52.888 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 16:58:53.352 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 16:58:53.355 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 16:58:53.360 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 16:58:53.364 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 16:58:53.802 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 16:58:53.809 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:58:53.965 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 16:58:53.970 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 16:58:53.970 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 16:58:54.335 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 16:58:54.416 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:58:54.819 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:58:54.846 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:58:54.851 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cef100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26783, start at 2024-05-31 16:58:30.250283822 +0800 CST m=+4.257961481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.256 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/31 16:58:55.287 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:58:55.295 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:58:55.320 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:58:55.751 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:58:55.772 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:58:55.780 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.17946.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139502484127745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139502484127745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:58:49 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.1798717989.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:58:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fa2df36-7ad5-4112-a6d9-dec2d5baa7de {"id":"8fa2df36-7ad5-4112-a6d9-dec2d5baa7de","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde79a519 8fa2df36-7ad5-4112-a6d9-dec2d5baa7de /tidb/cdc/default/default/upstream/7375084707672451587 {"id":7375084707672451587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fa2df36-7ad5-4112-a6d9-dec2d5baa7de {"id":"8fa2df36-7ad5-4112-a6d9-dec2d5baa7de","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde79a519 8fa2df36-7ad5-4112-a6d9-dec2d5baa7de /tidb/cdc/default/default/upstream/7375084707672451587 {"id":7375084707672451587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fa2df36-7ad5-4112-a6d9-dec2d5baa7de {"id":"8fa2df36-7ad5-4112-a6d9-dec2d5baa7de","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdde79a519 8fa2df36-7ad5-4112-a6d9-dec2d5baa7de /tidb/cdc/default/default/upstream/7375084707672451587 {"id":7375084707672451587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x lease 22318fcdde79a519 revoked table capture_session_done_during_task.t exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31384, start at 2024-05-31 16:58:13.947471558 +0800 CST m=+4.468300710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377bf0b00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:31477, start at 2024-05-31 16:58:13.954763041 +0800 CST m=+4.408712906 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:13.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:13.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:13.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:58:58 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3557335575.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully check diff failed 1-th time, retry later [2024/05/31 16:59:00.909 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/31 16:59:01.108 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:59:01.191 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:59:01.296 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 16:59:01.297 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:01.355 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:59:01.357 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:59:01.470 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:59:01.520 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:59:01.634 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:59:01.636 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:59:01.675 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:59:01.677 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:59:01.793 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 16:59:01.834 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:59:01 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [2024/05/31 16:59:01.987 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 16:59:01.992 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 16:59:02.044 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 16:59:02.046 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4184efd-2ffb-4459-964c-49c5ad3949d4 {"id":"f4184efd-2ffb-4459-964c-49c5ad3949d4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ea8e f4184efd-2ffb-4459-964c-49c5ad3949d4 /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4184efd-2ffb-4459-964c-49c5ad3949d4 {"id":"f4184efd-2ffb-4459-964c-49c5ad3949d4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ea8e f4184efd-2ffb-4459-964c-49c5ad3949d4 /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4184efd-2ffb-4459-964c-49c5ad3949d4 {"id":"f4184efd-2ffb-4459-964c-49c5ad3949d4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ea8e f4184efd-2ffb-4459-964c-49c5ad3949d4 /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/31 16:59:02.150 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:59:02.194 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 16:59:02.306 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 16:59:02.314 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 16:59:02.431 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 16:59:02.436 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 16:59:02.467 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:59:02.583 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:59:02.628 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:59:02.639 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:59:02.738 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:59:02.748 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 16:59:02.787 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:59:02.907 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:59:02.955 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 16:59:02.973 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 16:59:03.089 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:59:03.108 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 16:59:03.128 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 16:59:03.248 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 16:59:03.291 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 16:59:03.306 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 16:59:03.407 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 16:59:03.424 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 16:59:03.452 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 16:59:03.560 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 16:59:03.606 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 16:59:03.625 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] Verifying downstream PD is started... [2024/05/31 16:59:03.714 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 16:59:03.734 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 16:59:03.767 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 16:59:03.871 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 16:59:03.925 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 16:59:03.947 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 16:59:04.041 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 16:59:04.066 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 16:59:04.094 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later [2024/05/31 16:59:04.229 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 16:59:04.290 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 16:59:04.314 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 16:59:04.453 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 16:59:04.487 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 16:59:04.510 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 16:59:04.616 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 16:59:04.642 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 16:59:04.644 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 16:59:04.777 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 16:59:04.805 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 16:59:04.828 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 16:59:04.947 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/31 16:59:04.966 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 16:59:04.975 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/31 16:59:02.652 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/31 16:59:02.652 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/31 16:59:03.082 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/31 16:59:03.082 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/31 16:59:03.172 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd8b57bec_2b79_4c03_920f_869407d19e8e"] [2024/05/31 16:59:03.179 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs98ac59ab_1fba_4799_bb25_71611284c72d"] [2024/05/31 16:59:03.180 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb22c403b_91e8_4646_acc6_f089c96dbd35"] [2024/05/31 16:59:03.185 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse5b21d33_4da1_45e5_8c46_ee87d5610049"] [2024/05/31 16:59:03.198 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/31 16:59:03.201 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/31 16:59:03.228 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3986919f_f33c_4027_942d_8755574a83f3"] [2024/05/31 16:59:03.234 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs71033d6c_5cf5_4eb5_a22d_4ce96c49490f"] [2024/05/31 16:59:03.251 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs298cad2b_5f80_4ad3_93e2_26df574ee799"] [2024/05/31 16:59:03.260 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa97cf577_44e2_48ea_a40e_87f829d2d99b"] [2024/05/31 16:59:03.649 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/31 16:59:03.737 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/31 16:59:04.046 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.050 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.051 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.053 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.054 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.125 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.127 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.135 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.145 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.145 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.145 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.154 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.154 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.162 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.165 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:04.165 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:04.253 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/31 16:59:04.257 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/31 16:59:04.353 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/31 16:59:04.426 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/31 16:59:04.541 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/31 16:59:04.641 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/31 16:59:04.937 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.035 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.038 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.038 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.038 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.040 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.043 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.053 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.059 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.126 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.127 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.129 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.131 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.131 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:05.133 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.134 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:05.127 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/31 16:59:05.227 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/31 16:59:05.253 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:05.429 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/31 16:59:05.317 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/31 16:59:05.348 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/31 16:59:05.412 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/31 16:59:05.505 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] table test.finish_mark not exists for 3-th check, retry later [2024/05/31 16:59:05.451 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/31 16:59:05.543 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/31 16:59:05.544 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/31 16:59:05.598 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/31 16:59:05.629 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/31 16:59:05.696 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/31 16:59:05.727 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/31 16:59:05.837 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:05.944 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:05.946 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:05.946 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:05.951 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:05.787 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/31 16:59:06.028 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.032 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:06.042 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:06.043 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.045 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:06.048 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:06.131 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.133 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.135 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.141 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.142 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:06.349 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/31 16:59:06.553 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/31 16:59:06.834 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/31 16:59:06.844 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/31 16:59:06.857 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/31 16:59:06.928 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/31 16:59:06.940 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:06.946 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:06.950 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:06.951 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:07.032 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:07.037 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:07.054 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:07.054 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:07.057 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:07.133 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.147 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.242 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.253 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.256 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.257 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.257 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:07.445 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.finish_mark not exists for 4-th check, retry later [2024/05/31 16:59:07.555 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/31 16:59:07.926 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:07.931 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:07.936 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:07.938 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:07.941 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table kill_owner_with_ddl.t1 not exists for 1-th check, retry later [2024/05/31 16:59:08.028 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:08.035 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:08.039 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.042 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:08.046 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:08.048 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/31 16:59:08.055 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/31 16:59:08.133 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.234 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.244 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.327 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.328 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.328 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:08.449 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore [2024/05/31 16:59:08.543 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:08.835 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:08.845 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:08.859 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:08.927 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:08.938 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:08.938 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:08.948 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table kill_owner_with_ddl.t1 exists [2024/05/31 16:59:09.027 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:09.029 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:09.044 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.137 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.143 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.245 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.245 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.251 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:09.251 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.finish_mark not exists for 5-th check, retry later [2024/05/31 16:59:09.331 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/31 16:59:09.336 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/31 16:59:09.427 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/31 16:59:09.431 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/31 16:59:09.440 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24563) - No such process wait process 24563 exit for 1-th time... process 24563 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 16:59:09 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2483824840.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:09.539 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/31 16:59:09.753 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.839 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.855 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.928 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.932 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.932 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:09.937 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.027 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:10.031 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:10.046 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.137 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.140 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.243 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.244 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.333 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.337 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:10.442 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc7080016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25615, start at 2024-05-31 16:58:27.694337584 +0800 CST m=+4.572402510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cc6c40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:25718, start at 2024-05-31 16:58:27.636479936 +0800 CST m=+4.442157169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:27.643 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:27.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:27.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 16:59:10.545 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/31 16:59:10.652 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/31 16:59:10.738 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:59:10 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3011030112.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:10.832 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.847 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.848 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.851 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.929 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.931 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:10.956 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:10.956 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:11.026 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:11.103 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/31 16:59:11.284 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:59:11.053 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.144 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.227 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.237 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.251 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.342 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:59:11.441 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 16:59:11.443 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 16:59:11.494 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:59:11.495 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:59:11.331 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.331 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:11.445 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/31 16:59:11.546 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/31 16:59:11.597 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:59:11.649 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:59:11.760 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:59:11.762 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:59:11.746 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.752 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.finish_mark not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= [2024/05/31 16:59:11.831 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.806 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:59:11.836 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.806 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:59:11.839 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.917 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 16:59:11.846 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.958 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 16:59:11.927 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] The 1 times to try to start tidb cluster... [2024/05/31 16:59:11.935 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/31 16:59:11.938 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:11.940 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/31 16:59:11.947 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/31 16:59:12.026 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/31 16:59:12.035 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.055 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.143 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.230 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.246 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.247 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.327 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.331 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:12.439 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/31 16:59:12.086 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] table owner_remove_table_error.finished_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) Starting Upstream TiDB... ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) table owner_remove_table_error.finished_mark not exists for 2-th check, retry later table owner_remove_table_error.finished_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/31 16:59:12.087 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [Fri May 31 16:59:07 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> * Trying 127.0.0.1... Release Version: v6.5.9-17-g4afee51c07 Edition: Community * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 [2024/05/31 16:59:12.117 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] > Host: 127.0.0.1:8300 > Accept: */* > Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 [2024/05/31 16:59:12.118 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:12 GMT Git Branch: release-6.5 [2024/05/31 16:59:12.251 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 16:59:12.277 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:59:12.401 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' [2024/05/31 16:59:12.403 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 *** owner info ***: *** processors info ***: [2024/05/31 16:59:12.432 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] Race Enabled: false TiKV Min Version: 6.2.0-alpha changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 [2024/05/31 16:59:12.433 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc001d16240 TaskPositions:map[67ef601b-44ea-4a7d-861c-c15ee002d1b6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508345405446, checkpointTs: 450139508345405443, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67ef601b-44ea-4a7d-861c-c15ee002d1b6 {"id":"67ef601b-44ea-4a7d-861c-c15ee002d1b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf019b 67ef601b-44ea-4a7d-861c-c15ee002d1b6 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508345405443,"min-table-barrier-ts":450139508345405443,"admin-job-type":0} /tidb/cdc/default/default/task/position/67ef601b-44ea-4a7d-861c-c15ee002d1b6/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc001d16240 TaskPositions:map[67ef601b-44ea-4a7d-861c-c15ee002d1b6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508345405446, checkpointTs: 450139508345405443, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67ef601b-44ea-4a7d-861c-c15ee002d1b6 {"id":"67ef601b-44ea-4a7d-861c-c15ee002d1b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf019b 67ef601b-44ea-4a7d-861c-c15ee002d1b6 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508345405443,"min-table-barrier-ts":450139508345405443,"admin-job-type":0} /tidb/cdc/default/default/task/position/67ef601b-44ea-4a7d-861c-c15ee002d1b6/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc001d16240 TaskPositions:map[67ef601b-44ea-4a7d-861c-c15ee002d1b6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508345405446, checkpointTs: 450139508345405443, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67ef601b-44ea-4a7d-861c-c15ee002d1b6 {"id":"67ef601b-44ea-4a7d-861c-c15ee002d1b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf019b 67ef601b-44ea-4a7d-861c-c15ee002d1b6 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508345405443,"min-table-barrier-ts":450139508345405443,"admin-job-type":0} /tidb/cdc/default/default/task/position/67ef601b-44ea-4a7d-861c-c15ee002d1b6/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/31 16:59:12.542 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/31 16:59:12.744 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.828 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.829 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.838 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.838 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.848 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:12.937 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:13.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.138 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.155 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.243 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.244 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/31 16:59:13.244 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/31 16:59:13.244 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.340 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:13.348 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:12.552 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:59:12.586 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:59:12.707 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:59:12.709 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 16:59:12.733 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:59:12.734 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:59:12.855 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:59:12.880 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:59:13.007 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:59:13.009 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 16:59:13.033 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 16:59:13.034 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 16:59:13.196 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 16:59:13.220 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 16:59:13.350 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 16:59:13.352 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 16:59:13.373 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 16:59:13.374 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] table test.finish_mark not exists for 7-th check, retry later [2024/05/31 16:59:13.453 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/31 16:59:13.536 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/31 16:59:13.504 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 16:59:13.527 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 16:59:13.664 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 16:59:13.666 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 16:59:13.726 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.736 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.741 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.750 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.845 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.846 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:13.676 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 16:59:13.677 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 16:59:13.822 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 16:59:13.829 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 16:59:13.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:14.043 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.146 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:13.975 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 16:59:13.977 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 16:59:13.980 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 16:59:13.981 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 16:59:14.145 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 16:59:14.147 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:14.236 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.331 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.344 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.303 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 16:59:14.304 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 16:59:14.304 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 16:59:14.305 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cefb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26675, start at 2024-05-31 16:58:30.274373079 +0800 CST m=+4.332260344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377cef100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:26783, start at 2024-05-31 16:58:30.250283822 +0800 CST m=+4.257961481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:30.256 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:30.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:30.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 16:59:14 CST 2024] <<<<<< START cdc server in canal_json_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.3109031092.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:13 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 30115 owner id ec421db3-b475-49a4-8a17-0199f01c95b6 default [Fri May 31 16:59:14 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3026430266.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:14.432 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:14.531 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/31 16:59:14.540 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/31 16:59:14.556 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/31 16:59:14.628 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/31 16:59:14.629 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/31 16:59:14.633 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/31 16:59:14.640 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.453 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 16:59:14.457 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 16:59:14.606 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 16:59:14.607 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 16:59:14.610 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 16:59:14.612 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 16:59:14.731 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.735 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.736 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.740 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.832 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.837 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:14.754 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 16:59:14.759 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 16:59:14.904 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/31 16:59:14.905 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/31 16:59:14.912 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/31 16:59:14.913 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 16:59:15.051 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.148 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.056 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/31 16:59:15.063 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/31 16:59:15.208 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/31 16:59:15.209 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/31 16:59:15.240 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.330 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.332 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.349 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.431 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:15.222 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/31 16:59:15.224 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/31 16:59:15.360 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/31 16:59:15.376 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/31 16:59:15.534 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/31 16:59:15.550 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/31 16:59:15.627 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.507 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/31 16:59:15.508 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/31 16:59:15.522 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/31 16:59:15.524 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/31 16:59:15.654 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/31 16:59:15.669 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:15.728 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.730 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.738 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.742 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.841 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.847 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.847 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:15.926 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/31 16:59:15.952 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/31 16:59:15.799 +08:00] [INFO] [main.go:234] ["1 delete success: 1500"] [2024/05/31 16:59:15.800 +08:00] [INFO] [main.go:220] ["1 insert success: 3000"] [2024/05/31 16:59:15.815 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] [2024/05/31 16:59:15.817 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] [2024/05/31 16:59:15.947 +08:00] [INFO] [main.go:220] ["1 insert success: 3100"] [2024/05/31 16:59:15.966 +08:00] [INFO] [main.go:220] ["0 insert success: 3100"] table test.finish_mark not exists for 8-th check, retry later [2024/05/31 16:59:16.097 +08:00] [INFO] [main.go:234] ["1 delete success: 1600"] [2024/05/31 16:59:16.098 +08:00] [INFO] [main.go:220] ["1 insert success: 3200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/31 16:59:16.228 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.251 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.340 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.351 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.427 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.430 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.431 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.448 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:16.535 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/31 16:59:16.626 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.628 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/31 16:59:16.727 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.731 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.735 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.738 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.835 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.840 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:16.844 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:17.234 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.329 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/31 16:59:17.333 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/31 16:59:17.346 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/31 16:59:17.347 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.350 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/31 16:59:17.427 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.447 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26719, start at 2024-05-31 16:58:47.224147052 +0800 CST m=+4.273137115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... run test case test_kill_owner [Fri May 31 16:59:07 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4009640098.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.40158.out cli changefeed create --start-ts=450139492064428033 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info: {"upstream_id":7375085185851773928,"namespace":"default","id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T16:59:10.914970049+08:00","start_ts":450139492064428033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139492064428033,"checkpoint_ts":450139492064428033,"checkpoint_time":"2024-05-31 16:58:09.528"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 40101 owner id f1c553ce-b73d-4133-84bf-ca6ed70262b4 [Fri May 31 16:59:11 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4027040272.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0014e3248 TaskPositions:map[d3b6c44e-2690-4de4-9f12-ed6cd100425e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f1c553ce-b73d-4133-84bf-ca6ed70262b4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139509090680835, checkpointTs: 450139507202981919, state: Replicating tableID: 82, resolvedTs: 450139507202981919, checkpointTs: 450139507202981919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3b6c44e-2690-4de4-9f12-ed6cd100425e {"id":"d3b6c44e-2690-4de4-9f12-ed6cd100425e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59859 d3b6c44e-2690-4de4-9f12-ed6cd100425e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507202981919,"min-table-barrier-ts":450139507242303492,"admin-job-type":0} /tidb/cdc/default/default/task/position/d3b6c44e-2690-4de4-9f12-ed6cd100425e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f1c553ce-b73d-4133-84bf-ca6ed70262b4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0014e3248 TaskPositions:map[d3b6c44e-2690-4de4-9f12-ed6cd100425e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f1c553ce-b73d-4133-84bf-ca6ed70262b4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139509090680835, checkpointTs: 450139507202981919, state: Replicating tableID: 82, resolvedTs: 450139507202981919, checkpointTs: 450139507202981919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3b6c44e-2690-4de4-9f12-ed6cd100425e {"id":"d3b6c44e-2690-4de4-9f12-ed6cd100425e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59859 d3b6c44e-2690-4de4-9f12-ed6cd100425e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator"+ grep -q 'failed to get info:' :true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507202981919,"min-table-barrier-ts":450139507242303492,"admin-job-type":0} /tidb/cdc/default/default/task/position/d3b6c44e-2690-4de4-9f12-ed6cd100425e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f1c553ce-b73d-4133-84bf-ca6ed70262b4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0014e3248 TaskPositions:map[d3b6c44e-2690-4de4-9f12-ed6cd100425e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} f1c553ce-b73d-4133-84bf-ca6ed70262b4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139509090680835, checkpointTs: 450139507202981919, state: Replicating tableID: 82, resolvedTs: 450139507202981919, checkpointTs: 450139507202981919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3b6c44e-2690-4de4-9f12-ed6cd100425e {"id":"d3b6c44e-2690-4de4-9f12-ed6cd100425e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f1c553ce-b73d-4133-84bf-ca6ed70262b4 {"id":"f1c553ce-b73d-4133-84bf-ca6ed70262b4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59824 f1c553ce-b73d-4133-84bf-ca6ed70262b4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59859 d3b6c44e-2690-4de4-9f12-ed6cd100425e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507202981919,"min-table-barrier-ts":450139507242303492,"admin-job-type":0} /tidb/cdc/default/default/task/position/d3b6c44e-2690-4de4-9f12-ed6cd100425e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f1c553ce-b73d-4133-84bf-ca6ed70262b4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f1c553ce-b73d-4133-84bf-ca6ed70262b4" | grep -v cluster_id | grep id "cluster-id": "default" "id": "d3b6c44e-2690-4de4-9f12-ed6cd100425e", "cluster-id": "default" run task successfully capture_id: d3b6c44e-2690-4de4-9f12-ed6cd100425e /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (40101) - No such process wait process 40101 exit for 1-th time... process 40101 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep d3b6c44e-2690-4de4-9f12-ed6cd100425e -A1 | grep '"is-owner": true' "is-owner": true, run task successfully test_kill_owner: pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_owner [Fri May 31 16:59:16 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4047340475.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:17.534 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:17.539 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:17.545 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/31 16:59:17.641 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/31 16:59:17.730 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:17.739 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:17.747 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:17 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: + (( i++ )) + (( i <= 50 )) /tidb/cdc/default/__cdc_meta__/capture/78413ce1-5891-40f5-a48e-6be94b27197f ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info {"id":"78413ce1-5891-40f5-a48e-6be94b27197f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfc01a8e 78413ce1-5891-40f5-a48e-6be94b27197f > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK /tidb/cdc/default/default/upstream/7375085495036263917 < Date: Fri, 31 May 2024 08:59:17 GMT < Content-Length: 911 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: {"id":7375085495036263917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** etcd info ***: *** processors info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e *** etcd info ***: {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/__cdc_meta__/capture/78413ce1-5891-40f5-a48e-6be94b27197f {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 {"id":"78413ce1-5891-40f5-a48e-6be94b27197f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfc01a8e 78413ce1-5891-40f5-a48e-6be94b27197f /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/default/upstream/7375085495036263917 /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085495036263917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** owner info ***: *** processors info ***: *** etcd info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/__cdc_meta__/capture/78413ce1-5891-40f5-a48e-6be94b27197f {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} {"id":"78413ce1-5891-40f5-a48e-6be94b27197f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfc01a8e {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} 78413ce1-5891-40f5-a48e-6be94b27197f /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/default/upstream/7375085495036263917 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 {"id":7375085495036263917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.cli.31156.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Create changefeed successfully! {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} ID: babca023-9d46-4d31-bc5f-c89bfeba5ec6 /tidb/cdc/default/__cdc_meta__/capture/ec421db3-b475-49a4-8a17-0199f01c95b6 {"id":"ec421db3-b475-49a4-8a17-0199f01c95b6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba38 ec421db3-b475-49a4-8a17-0199f01c95b6 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "ec421db3-b475-49a4-8a17-0199f01c95b6 default" | grep id "id": "e9e9ea5d-d031-4605-91cb-116c27e1075e", run task successfully Info: {"upstream_id":7375085495036263917,"namespace":"default","id":"babca023-9d46-4d31-bc5f-c89bfeba5ec6","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-31T16:59:17.769594517+08:00","start_ts":450139509922463748,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139509922463748,"checkpoint_ts":450139509922463748,"checkpoint_time":"2024-05-31 16:59:17.651"} PASS table test.finish_mark not exists for 9-th check, retry later [2024/05/31 16:59:17.841 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:17.848 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x capture_id: e9e9ea5d-d031-4605-91cb-116c27e1075e pass test: check health /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24843) - No such process wait process 24843 exit for 1-th time... process 24843 already exit check_capture_count http://127.0.0.1:2379 0 [2024/05/31 16:59:18.352 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:18.449 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:18.454 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:18.528 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:18.537 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:18.556 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:18.557 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] pass test: get status table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 16:59:18 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2506825070.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:18.627 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:18.628 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/31 16:59:18.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:18.646 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:18.650 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] pass test: create changefeed [2024/05/31 16:59:19.144 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:19.245 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:19.253 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:19.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:19.330 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:19.333 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:19.335 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:19.339 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:19.357 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/31 16:59:19.357 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:19.358 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:19.561 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/31 16:59:19.565 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.finish_mark not exists for 10-th check, retry later [2024/05/31 16:59:19.772 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/31 16:59:19.786 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/31 16:59:19.786 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/31 16:59:19.833 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/31 16:59:19.839 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/31 16:59:20.072 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/05/31 16:59:20.167 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/31 16:59:20.174 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc002c2a120 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450139507242303492, checkpointTs: 450139507242303492, state: Preparing tableID: 82, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating tableID: 80, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507242303492,"min-table-barrier-ts":450139510401400835,"admin-job-type":0} /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc002c2a120 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450139507242303492, checkpointTs: 450139507242303492, state: Preparing tableID: 82, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating tableID: 80, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507242303492,"min-table-barrier-ts":450139510401400835,"admin-job-type":0} /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc002c2a120 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 450139507242303492, checkpointTs: 450139507242303492, state: Preparing tableID: 82, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating tableID: 80, resolvedTs: 450139510401400835, checkpointTs: 450139510401400835, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139507242303492,"min-table-barrier-ts":450139510401400835,"admin-job-type":0} /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/31 16:59:20.180 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/31 16:59:20.254 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/31 16:59:20.261 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/31 16:59:20.353 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8ce2912c_6f77_47df_ab9b_fa95954d74dd"] table ddl_attributes.finish_mark not exists for 3-th check, retry later "is-owner": true, run task successfully [2024/05/31 16:59:20.438 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs417905aa_f0b7_4408_8f45_0206f0670d75"] [2024/05/31 16:59:20.445 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc179bd75_9c46_433f_b2cb_0793dea989b8"] [2024/05/31 16:59:20.454 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs569939e0_daad_4db7_8187_e99a83695f3b"] [2024/05/31 16:59:20.538 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8e10ad6e_cfd4_445f_aa09_76851f65ed1e"] [2024/05/31 16:59:20.539 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs03745c40_7e96_4687_8f4c_107497a3160e"] owner pid: 40478 owner id 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 [Fri May 31 16:59:20 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4061440616.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:20.742 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/31 16:59:20.746 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/05/31 16:59:20.943 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/31 16:59:21.143 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:21.232 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.238 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:21.246 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:21.331 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.331 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.338 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.342 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:21.346 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:21.354 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:21.428 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.429 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:21.406 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] table test.finish_mark not exists for 11-th check, retry later [2024/05/31 16:59:21.598 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:59:21.601 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:59:21.757 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:59:21.759 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 16:59:21.759 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:59:21.760 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0014df0e0 TaskPositions:map[e9c0f54a-35de-4bdb-bf2c-c81081b5a612:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c0f54a-35de-4bdb-bf2c-c81081b5a612 {"id":"e9c0f54a-35de-4bdb-bf2c-c81081b5a612","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf01e6 e9c0f54a-35de-4bdb-bf2c-c81081b5a612 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/e9c0f54a-35de-4bdb-bf2c-c81081b5a612/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0014df0e0 TaskPositions:map[e9c0f54a-35de-4bdb-bf2c-c81081b5a612:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c0f54a-35de-4bdb-bf2c-c81081b5a612 {"id":"e9c0f54a-35de-4bdb-bf2c-c81081b5a612","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf01e6 e9c0f54a-35de-4bdb-bf2c-c81081b5a612 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/e9c0f54a-35de-4bdb-bf2c-c81081b5a612/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0014df0e0 TaskPositions:map[e9c0f54a-35de-4bdb-bf2c-c81081b5a612:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9c0f54a-35de-4bdb-bf2c-c81081b5a612 {"id":"e9c0f54a-35de-4bdb-bf2c-c81081b5a612","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf01e6 e9c0f54a-35de-4bdb-bf2c-c81081b5a612 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/e9c0f54a-35de-4bdb-bf2c-c81081b5a612/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/31 16:59:21.836 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/31 16:59:21.934 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/05/31 16:59:21.946 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:21.918 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:59:21.923 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:59:22.027 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdead6fac_04df_411f_8b35_91b5289eb21b"] [2024/05/31 16:59:22.127 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.128 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:22.129 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:22.157 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.227 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:22.229 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.235 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:22.237 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.245 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:22.078 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:59:22.079 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:59:22.082 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:59:22.083 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:59:22.241 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 16:59:22.247 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table ddl_attributes.finish_mark exists check diff successfully [2024/05/31 16:59:22.259 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.334 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:22.408 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 16:59:22.410 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 16:59:22.412 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 16:59:22.413 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 16:59:22.534 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa8b132bd_d33a_4cf4_94dd_be6cb1d13f51"] [2024/05/31 16:59:22.554 +08:00] [INFO] [main.go:178] ["1 insert success: 2300"] [2024/05/31 16:59:22.737 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:22.572 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:59:22.577 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 16:59:22.736 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 16:59:22.738 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 16:59:22.738 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 16:59:22.739 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:59:22 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> [2024/05/31 16:59:22.926 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:22.927 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:22.927 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:22.928 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:22.955 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:22.897 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:59:22.900 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:59:23.031 +08:00] [INFO] [main.go:178] ["0 insert success: 1800"] [2024/05/31 16:59:23.032 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:23.034 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:23.035 +08:00] [INFO] [main.go:199] ["0 delete success: 900"] [2024/05/31 16:59:23.037 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:23.042 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:23.045 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:23.147 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:23.154 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:23.228 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:23.061 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:59:23.063 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:59:23.063 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 16:59:23.064 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:59:23.230 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:59:23.233 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:59:23.429 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:23.434 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:23.526 +08:00] [INFO] [main.go:178] ["1 insert success: 2400"] table test.finish_mark not exists for 12-th check, retry later [2024/05/31 16:59:23.395 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 16:59:23.397 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 16:59:23.397 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 16:59:23.398 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:59:23.553 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 16:59:23.558 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 16:59:23.547 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:23.719 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 16:59:23.721 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 16:59:23.726 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 16:59:23.727 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:23.877 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 16:59:23.889 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 16:59:24.030 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 16:59:24.031 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 16:59:24.040 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 16:59:24.041 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0011bdbf0 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b23867e3-96ba-42c0-bfeb-67e1f79d22dd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/b23867e3-96ba-42c0-bfeb-67e1f79d22dd {"id":"b23867e3-96ba-42c0-bfeb-67e1f79d22dd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598ad b23867e3-96ba-42c0-bfeb-67e1f79d22dd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511187832837,"min-table-barrier-ts":450139511449714693,"admin-job-type":0} /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b23867e3-96ba-42c0-bfeb-67e1f79d22dd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0011bdbf0 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b23867e3-96ba-42c0-bfeb-67e1f79d22dd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/b23867e3-96ba-42c0-bfeb-67e1f79d22dd {"id":"b23867e3-96ba-42c0-bfeb-67e1f79d22dd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598ad b23867e3-96ba-42c0-bfeb-67e1f79d22dd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use+ grep -q 'failed to get info:' -file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511187832837,"min-table-barrier-ts":450139511449714693,"admin-job-type":0} /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b23867e3-96ba-42c0-bfeb-67e1f79d22dd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0011bdbf0 TaskPositions:map[3175a6d8-8610-4ac8-968b-c99d61cbd3d5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b23867e3-96ba-42c0-bfeb-67e1f79d22dd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3175a6d8-8610-4ac8-968b-c99d61cbd3d5 {"id":"3175a6d8-8610-4ac8-968b-c99d61cbd3d5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/b23867e3-96ba-42c0-bfeb-67e1f79d22dd {"id":"b23867e3-96ba-42c0-bfeb-67e1f79d22dd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5988a 3175a6d8-8610-4ac8-968b-c99d61cbd3d5 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598ad b23867e3-96ba-42c0-bfeb-67e1f79d22dd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511187832837,"min-table-barrier-ts":450139511449714693,"admin-job-type":0} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/3175a6d8-8610-4ac8-968b-c99d61cbd3d5/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b23867e3-96ba-42c0-bfeb-67e1f79d22dd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "3175a6d8-8610-4ac8-968b-c99d61cbd3d5" | grep id "cluster-id": "default" "id": "b23867e3-96ba-42c0-bfeb-67e1f79d22dd", "cluster-id": "default" run task successfully [2024/05/31 16:59:24.184 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 16:59:24.201 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 16:59:24.335 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 16:59:24.336 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 16:59:24.228 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:24.233 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:24.234 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:24.351 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 16:59:24.352 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 16:59:24.487 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 16:59:24.507 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 16:59:24.428 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:24.533 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] capture_id: b23867e3-96ba-42c0-bfeb-67e1f79d22dd ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'b23867e3-96ba-42c0-bfeb-67e1f79d22dd' b23867e3-96ba-42c0-bfeb-67e1f79d22dd run task successfully test_hang_up_owner: pass [2024/05/31 16:59:24.636 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 16:59:24.637 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 16:59:24.655 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 16:59:24.655 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 16:59:24.785 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 16:59:24.805 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 16:59:24.632 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:24.639 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:24.729 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:24.734 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:24.747 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:24.937 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 16:59:24.938 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 16:59:24.957 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 16:59:24.958 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 16:59:25.103 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 16:59:24.936 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:25.039 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:25.046 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_owner [Fri May 31 16:59:24 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4077240774.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:25.144 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:25.337 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.345 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:25.124 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 16:59:25.263 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/31 16:59:25.264 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/31 16:59:25.279 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 16:59:25.280 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/31 16:59:25.550 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.552 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:25.422 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/31 16:59:25.441 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/31 16:59:25.583 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/31 16:59:25.585 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/31 16:59:25.596 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/31 16:59:25.597 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032340003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19240, start at 2024-05-31 16:59:23.663696097 +0800 CST m=+4.391989589 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.672 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log [2024/05/31 16:59:25.626 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.635 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.729 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.735 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:25.757 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:25.737 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/31 16:59:25.750 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] table test.finish_mark not exists for 13-th check, retry later [2024/05/31 16:59:25.939 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:25.939 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:25.940 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:26.042 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:26.052 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:25.887 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/31 16:59:25.888 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/31 16:59:25.900 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/31 16:59:25.901 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/31 16:59:26.037 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/31 16:59:26.050 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] [2024/05/31 16:59:26.131 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:26.135 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:26.189 +08:00] [INFO] [main.go:220] ["1 insert success: 3000"] [2024/05/31 16:59:26.191 +08:00] [INFO] [main.go:234] ["1 delete success: 1500"] [2024/05/31 16:59:26.201 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] [2024/05/31 16:59:26.202 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] [2024/05/31 16:59:26.340 +08:00] [INFO] [main.go:220] ["1 insert success: 3100"] [2024/05/31 16:59:26.354 +08:00] [INFO] [main.go:220] ["0 insert success: 3100"] [2024/05/31 16:59:26.425 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:26.534 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:26.736 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:26.743 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:26.751 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:26.830 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:26.850 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:26.853 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:27.031 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.146 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:27.146 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.148 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.152 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.253 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.427 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:27.427 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:27.818 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table test.finish_mark not exists for 14-th check, retry later [2024/05/31 16:59:28.251 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:28.336 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:28.350 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:28.428 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:28.436 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:28.457 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000774a98 TaskPositions:map[3c52f812-e7fa-407f-864d-1b1a62204c39:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 84, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 82, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c52f812-e7fa-407f-864d-1b1a62204c39 {"id":"3c52f812-e7fa-407f-864d-1b1a62204c39","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598cf 3c52f812-e7fa-407f-864d-1b1a62204c39 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511449714693,"min-table-barrier-ts":450139512498290692,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c52f812-e7fa-407f-864d-1b1a62204c39/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000774a98 TaskPositions:map[3c52f812-e7fa-407f-864d-1b1a62204c39:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 84, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 82, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c52f812-e7fa-407f-864d-1b1a62204c39 {"id":"3c52f812-e7fa-407f-864d-1b1a62204c39","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598cf 3c52f812-e7fa-407f-864d-1b1a62204c39 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511449714693,"min-table-barrier-ts":450139512498290692,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c52f812-e7fa-407f-864d-1b1a62204c39/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000774a98 TaskPositions:map[3c52f812-e7fa-407f-864d-1b1a62204c39:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 84, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing tableID: 82, resolvedTs: 450139511449714693, checkpointTs: 450139511449714693, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c52f812-e7fa-407f-864d-1b1a62204c39 {"id":"3c52f812-e7fa-407f-864d-1b1a62204c39","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598cf 3c52f812-e7fa-407f-864d-1b1a62204c39 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentag+ grep -q 'etcd info' e":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139511449714693,"min-table-barrier-ts":450139512498290692,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c52f812-e7fa-407f-864d-1b1a62204c39/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 16:59:28.529 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] owner pid: 40777 owner id 3c52f812-e7fa-407f-864d-1b1a62204c39 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '3c52f812-e7fa-407f-864d-1b1a62204c39' /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598cf run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/31 16:59:28.733 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.737 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:28.751 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:28.832 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.836 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.839 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.840 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.841 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:28.847 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:29.142 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 26.490160128s"] "is-owner": true, run task successfully test_expire_owner pass [2024/05/31 16:59:29.231 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:29.240 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:29.330 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:29.336 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:29.346 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:29.436 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:29.439 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:29.645 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:29.727 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check_changefeed_status 127.0.0.1:8300 e92f30a1-e1be-4e0c-a56f-e460df33e7a5 normal last_error null + endpoint=127.0.0.1:8300 start tidb cluster in /tmp/tidb_cdc_test/common_1 table test.finish_mark not exists for 15-th check, retry later Starting Upstream PD... + changefeed_id=e92f30a1-e1be-4e0c-a56f-e460df33e7a5 + expected_state=normal Release Version: v6.5.9-2-g33d839bdd Edition: Community + field=last_error + error_pattern=null Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 ++ curl 127.0.0.1:8300/api/v2/changefeeds/e92f30a1-e1be-4e0c-a56f-e460df33e7a5/status UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... % Total % Received % Xferd Average Speed Time Time Time Current Release Version: v6.5.9-2-g33d839bdd Edition: Community Dload Upload Total Spent Left Speed Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 861 0 --:--:-- --:--:-- --:--:-- 868 + info='{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' + echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' {"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180} ++ echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 e92f30a1-e1be-4e0c-a56f-e460df33e7a5 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=e92f30a1-e1be-4e0c-a56f-e460df33e7a5 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/e92f30a1-e1be-4e0c-a56f-e460df33e7a5/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1033 0 --:--:-- --:--:-- --:--:-- 1036 + info='{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' + echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' {"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180} ++ echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":450139512673927180,"checkpoint_ts":450139512673927180}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Fri May 31 16:59:29 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4096940971.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:29.742 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:29.851 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:29.925 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:29.931 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:29.938 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:29.939 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:29.945 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:30.045 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:30.133 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:30.141 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:30.145 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:30.231 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:30.338 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:30.339 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:30.730 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:30.746 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:30.752 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:30.951 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:30.955 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later [2024/05/31 16:59:31.031 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:31.031 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:31.040 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:31.051 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:31.054 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:31.149 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:31.150 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:31.225 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:31.227 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:31.256 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:31.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 16-th check, retry later [2024/05/31 16:59:31.531 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:31.537 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:31.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (25073) - No such process wait process 25073 exit for 1-th time... process 25073 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 16:59:30 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2527125273.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:31.948 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:32.026 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:32.027 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:32.028 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:32.031 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:32.038 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:32.045 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:32.046 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:32.053 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:32.127 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:32.144 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:32.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:32.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:32.431 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:32.438 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:32.534 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:31.792 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/31 16:59:31.985 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:59:32.042 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:59:32.136 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 16:59:32.138 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 16:59:32.194 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:59:32.195 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:59:32.300 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:59:32.343 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:59:32.464 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:59:32.466 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:59:32.502 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:59:32.503 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:59:32.622 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 16:59:32.659 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 16:59:32.774 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 16:59:32.775 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 16:59:32.815 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 16:59:32.816 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 16:59:32.928 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/31 16:59:32.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:32.944 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:33.028 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:33.031 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:33.033 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:33.038 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table sink_hang.t2 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000ca9890 TaskPositions:map[ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 82, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 84, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139512760696836,"min-table-barrier-ts":450139513809272836,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000ca9890 TaskPositions:map[ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 82, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 84, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139512760696836,"min-table-barrier-ts":450139513809272836,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000ca9890 TaskPositions:map[ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 82, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing tableID: 84, resolvedTs: 450139512760696836, checkpointTs: 450139512760696836, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139512760696836,"min-table-barrier-ts":450139513809272836,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully [2024/05/31 16:59:32.965 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:59:33.080 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 16:59:33.082 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 16:59:33.122 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 16:59:33.124 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 16:59:33.046 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:33.046 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:33.051 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:33.055 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:33.150 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:33.227 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:33.233 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0008be4f8 TaskPositions:map[8732b12c-beaa-443d-855f-536dc1d71f2c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8732b12c-beaa-443d-855f-536dc1d71f2c {"id":"8732b12c-beaa-443d-855f-536dc1d71f2c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf021b 8732b12c-beaa-443d-855f-536dc1d71f2c /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/8732b12c-beaa-443d-855f-536dc1d71f2c/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0008be4f8 TaskPositions:map[8732b12c-beaa-443d-855f-536dc1d71f2c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8732b12c-beaa-443d-855f-536dc1d71f2c {"id":"8732b12c-beaa-443d-855f-536dc1d71f2c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf021b 8732b12c-beaa-443d-855f-536dc1d71f2c /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/8732b12c-beaa-443d-855f-536dc1d71f2c/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc0008be4f8 TaskPositions:map[8732b12c-beaa-443d-855f-536dc1d71f2c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8732b12c-beaa-443d-855f-536dc1d71f2c {"id":"8732b12c-beaa-443d-855f-536dc1d71f2c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf021b 8732b12c-beaa-443d-855f-536dc1d71f2c /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/8732b12c-beaa-443d-855f-536dc1d71f2c/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x owner pid: 40974 owner id ee60f605-e698-4d50-80f0-096a6fc8958e [Fri May 31 16:59:33 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4111841120.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c88001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18369, start at 2024-05-31 16:59:30.515258692 +0800 CST m=+5.275935862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [2024/05/31 16:59:33.270 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:59:33.305 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:59:33.431 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:59:33.432 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 16:59:33.425 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:33.433 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] check_capture_count http://127.0.0.1:2379 1 run task successfully table test.finish_mark not exists for 17-th check, retry later TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log [2024/05/31 16:59:33.459 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:59:33.461 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:59:33.589 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:59:33.618 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:59:33.550 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:33.736 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:33.739 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:59:33.740 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 16:59:33.781 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 16:59:33.783 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 16:59:33.898 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 16:59:33.946 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 16:59:33.832 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:33.928 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:33.936 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:33.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:33.940 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:34.028 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:34.030 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:34.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:34.053 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 16:59:34.055 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 16:59:34.109 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 16:59:34.111 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 16:59:34.214 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 16:59:34.126 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:34.136 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:34.139 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:34.234 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:34.264 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 16:59:34.371 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 16:59:34.372 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 16:59:34.420 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 16:59:34.421 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 16:59:34.344 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:34.526 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 16:59:34.572 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 16:59:34.678 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 16:59:34.679 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 16:59:34.725 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 16:59:34.727 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 16:59:34.647 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:34.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:34.743 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:34.743 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:34.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:34.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:34.763 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:34.828 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 16:59:34.874 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 16:59:34.979 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 16:59:34.980 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 16:59:34.832 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:34.841 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:34.847 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:34.848 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:34.866 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:34.953 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:35.026 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 16:59:35.028 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 16:59:35.154 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 16:59:35.205 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 16:59:35.311 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 16:59:35.312 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 16:59:35.356 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 16:59:35.358 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 16:59:35.460 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] Starting CDC state checker... check diff failed 2-th time, retry later [2024/05/31 16:59:35.362 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:35.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:35.534 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:35.537 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:35.538 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:35.557 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:35.508 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 16:59:35.613 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/31 16:59:35.614 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 16:59:35.659 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/31 16:59:35.660 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/31 16:59:35.645 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:35.649 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:35.725 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:35.726 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:35.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:35.750 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:35.765 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/31 16:59:35.812 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/31 16:59:35.916 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/31 16:59:35.917 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/31 16:59:35.960 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/31 16:59:35.961 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] table test.finish_mark exists check diff successfully [2024/05/31 16:59:36.067 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/31 16:59:36.113 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/31 16:59:36.222 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/31 16:59:36.224 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:59:36 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/31 16:59:36.232 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5d43c426_fbf4_4fa2_88b4_9a9239018a59"] [2024/05/31 16:59:36.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:36.332 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:36.434 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:36.439 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 16:59:36.269 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/31 16:59:36.270 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/31 16:59:36.375 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] [2024/05/31 16:59:36.424 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/31 16:59:36.521 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] [2024/05/31 16:59:36.522 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037eeee8 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514582597637,"min-table-barrier-ts":450139514595442695,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037eeee8 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514582597637,"min-table-barrier-ts":450139514595442695,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037eeee8 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514582597637,"min-table-barrier-ts":450139514595442695,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "ee60f605-e698-4d50-80f0-096a6fc8958e" | grep id "cluster-id": "default" "id": "00398d19-a9e2-41dd-b9f3-ab7e443901cd", "cluster-id": "default" run task successfully [2024/05/31 16:59:36.460 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:36.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:36.570 +08:00] [INFO] [main.go:220] ["1 insert success: 3000"] [2024/05/31 16:59:36.571 +08:00] [INFO] [main.go:234] ["1 delete success: 1500"] [2024/05/31 16:59:36.669 +08:00] [INFO] [main.go:220] ["0 insert success: 3100"] [2024/05/31 16:59:36.720 +08:00] [INFO] [main.go:220] ["1 insert success: 3100"] [2024/05/31 16:59:36.731 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:36.745 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:36.849 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:36.977 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:37.041 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:37.054 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:37.146 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] capture_id: 00398d19-a9e2-41dd-b9f3-ab7e443901cd [2024/05/31 16:59:37.227 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf98908b0_02df_4f53_9c9b_aa0769829a74"] [2024/05/31 16:59:37.236 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:37.327 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsee505865_8e14_4fa1_a5b7_7b1ac069b2e6"] [2024/05/31 16:59:37.347 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] check diff failed 3-th time, retry later [2024/05/31 16:59:37.630 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:37.735 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 16:59:37.750 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:37.837 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:37.928 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:37.930 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:37.934 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:37.946 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9802ea2b_ad8c_4994_bd9f_820fc8227512"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:38.033 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:38.033 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:38.126 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:38.137 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa323f678_341f_4e81_bbcf_7dfd1eef7f89"] [2024/05/31 16:59:38.429 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs197ac662_9270_4524_b039_b7de6bc577c1"] start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_stop_delay.t not exists for 1-th check, retry later [2024/05/31 16:59:38.828 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 16:59:38.855 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:38.856 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:38.934 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:38.954 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:38.957 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:38.959 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:38.961 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:39.043 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:39.047 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:39.128 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:39.225 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26621, start at 2024-05-31 16:58:47.218774314 +0800 CST m=+4.330888855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f377df8740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:26719, start at 2024-05-31 16:58:47.224147052 +0800 CST m=+4.273137115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:00:47.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:58:47.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:48:47.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.30783.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 16:59:39.332 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:39.531 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table processor_stop_delay.t exists check diff failed 1-th time, retry later + set +x + tso='450139515583463425 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139515583463425 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 16:59:39 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.3082830830.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:39.829 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:39.853 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:39.929 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:39.930 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:39.941 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:39.942 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:39.942 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:39.956 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:39.959 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:39.969 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:40.036 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:40.226 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] 0 [Fri May 31 16:59:40 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4127641278.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:40.546 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:40.637 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:40.658 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:40.730 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:40.739 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:40.746 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs86333329_8ab6_436f_8359_663c8382478a"] [2024/05/31 16:59:40.827 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:40.830 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:40.835 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:40.841 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:40.841 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:40.845 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:40.948 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:41.229 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:41.254 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:41.341 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:41.347 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:41.431 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:41.443 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:41.529 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:41.633 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:41.644 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:41.645 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:41.726 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:41.735 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:41.735 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:41.747 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8f7d959a_be59_4f13_98f8_8ca508575f44"] [2024/05/31 16:59:41.854 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later [2024/05/31 16:59:41.952 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/31 16:59:42.128 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff successfully [2024/05/31 16:59:42.133 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 16:59:42.134 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 16:59:42.284 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 16:59:42.285 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 16:59:42.286 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 16:59:42.287 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 16:59:42.229 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:42.244 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:42.251 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:42.328 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:42.344 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:42.447 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 16:59:42.448 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 16:59:42.434 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:42.445 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:42.456 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:42.541 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:42.544 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:42.545 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:42.548 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:42.552 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:42.638 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 16:59:42.639 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 16:59:42.646 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 16:59:42.647 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 16:59:42.731 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:42.740 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1dab64c-27de-405d-82ab-b2fbd0f9810c {"id":"c1dab64c-27de-405d-82ab-b2fbd0f9810c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde01ebf8e c1dab64c-27de-405d-82ab-b2fbd0f9810c /tidb/cdc/default/default/upstream/7375085568027697189 {"id":7375085568027697189,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1dab64c-27de-405d-82ab-b2fbd0f9810c {"id":"c1dab64c-27de-405d-82ab-b2fbd0f9810c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde01ebf8e c1dab64c-27de-405d-82ab-b2fbd0f9810c /tidb/cdc/default/default/upstream/7375085568027697189 {"id":7375085568027697189,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1dab64c-27de-405d-82ab-b2fbd0f9810c {"id":"c1dab64c-27de-405d-82ab-b2fbd0f9810c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde01ebf8e c1dab64c-27de-405d-82ab-b2fbd0f9810c /tidb/cdc/default/default/upstream/7375085568027697189 {"id":7375085568027697189,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.30905.out cli changefeed create --start-ts=450139515583463425 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: aeb09f31-cc33-443f-8ded-ef58e6cd372a Info: {"upstream_id":7375085568027697189,"namespace":"default","id":"aeb09f31-cc33-443f-8ded-ef58e6cd372a","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-31T16:59:42.940275892+08:00","start_ts":450139515583463425,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139515583463425,"checkpoint_ts":450139515583463425,"checkpoint_time":"2024-05-31 16:59:39.246"} PASS [2024/05/31 16:59:42.866 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 16:59:42.928 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 16:59:42.936 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:43.141 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:43.142 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [2024/05/31 16:59:43.123 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 16:59:43.126 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 16:59:43.222 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 16:59:43.223 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 16:59:43.238 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:43.326 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:43.344 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (25276) - No such process wait process 25276 exit for 1-th time... process 25276 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 16:59:42 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2548225486.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:43.432 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 16:59:43.485 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff failed 1-th time, retry later table changefeed_pause_resume.t1 not exists for 1-th check, retry later table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later [2024/05/31 16:59:43.449 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:43.530 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:43.631 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:43.634 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:43.648 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:43.650 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:43.653 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc00077ce70 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c81913ea-5dfb-47f2-ad15-d9cb283ac51a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c81913ea-5dfb-47f2-ad15-d9cb283ac51a {"id":"c81913ea-5dfb-47f2-ad15-d9cb283ac51a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5995a c81913ea-5dfb-47f2-ad15-d9cb283ac51a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514844741639,"min-table-barrier-ts":450139514857586696,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c81913ea-5dfb-47f2-ad15-d9cb283ac51a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc00077ce70 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c81913ea-5dfb-47f2-ad15-d9cb283ac51a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c81913ea-5dfb-47f2-ad15-d9cb283ac51a {"id":"c81913ea-5dfb-47f2-ad15-d9cb283ac51a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5995a c81913ea-5dfb-47f2-ad15-d9cb283ac51a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514844741639,"min-table-barrier-ts":450139514857586696,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c81913ea-5dfb-47f2-ad15-d9cb283ac51a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc00077ce70 TaskPositions:map[00398d19-a9e2-41dd-b9f3-ab7e443901cd:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} c81913ea-5dfb-47f2-ad15-d9cb283ac51a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ee60f605-e698-4d50-80f0-096a6fc8958e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00398d19-a9e2-41dd-b9f3-ab7e443901cd {"id":"00398d19-a9e2-41dd-b9f3-ab7e443901cd","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/c81913ea-5dfb-47f2-ad15-d9cb283ac51a {"id":"c81913ea-5dfb-47f2-ad15-d9cb283ac51a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ee60f605-e698-4d50-80f0-096a6fc8958e {"id":"ee60f605-e698-4d50-80f0-096a6fc8958e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec598fb ee60f605-e698-4d50-80f0-096a6fc8958e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5991c 00398d19-a9e2-41dd-b9f3-ab7e443901cd /tidb/cdc/default/__cdc_meta__/owner/22318fcddec5995a c81913ea-5dfb-47f2-ad15-d9cb283ac51a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139514844741639,"min-table-barrier-ts":450139514857586696,"admin-job-type":0} /tidb/cdc/default/default/task/position/00398d19-a9e2-41dd-b9f3-ab7e443901cd/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c81913ea-5dfb-47f2-ad15-d9cb283ac51a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee60f605-e698-4d50-80f0-096a6fc8958e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:43.707 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 16:59:43.710 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 16:59:43.760 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 16:59:43.762 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] check diff failed 6-th time, retry later [2024/05/31 16:59:43.725 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:43.750 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:43.755 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:43.837 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:43.968 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 16:59:44.000 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 16:59:43.930 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:43.932 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:44.030 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:44.136 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:44.181 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 16:59:44.184 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 16:59:44.236 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 16:59:44.236 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 16:59:44.237 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:44.353 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table changefeed_pause_resume.t2 not exists for 2-th check, retry later [2024/05/31 16:59:44.422 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 16:59:44.478 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 16:59:44.442 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:44.535 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:44.547 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:44.625 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:44.634 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:44.642 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:44.643 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:44.778 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 16:59:44.783 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 16:59:44.825 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 16:59:44.826 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 16:59:44.742 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:44.747 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:44.753 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:44.848 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:44.850 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:44.931 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:44.949 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:45.039 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:45 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc000f1f008 TaskPositions:map[fdcde237-80a1-408d-b137-6275ab52f97d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdcde237-80a1-408d-b137-6275ab52f97d {"id":"fdcde237-80a1-408d-b137-6275ab52f97d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf025a fdcde237-80a1-408d-b137-6275ab52f97d /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/fdcde237-80a1-408d-b137-6275ab52f97d/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc000f1f008 TaskPositions:map[fdcde237-80a1-408d-b137-6275ab52f97d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdcde237-80a1-408d-b137-6275ab52f97d {"id":"fdcde237-80a1-408d-b137-6275ab52f97d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf025a fdcde237-80a1-408d-b137-6275ab52f97d /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/fdcde237-80a1-408d-b137-6275ab52f97d/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc000f1f008 TaskPositions:map[fdcde237-80a1-408d-b137-6275ab52f97d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdcde237-80a1-408d-b137-6275ab52f97d {"id":"fdcde237-80a1-408d-b137-6275ab52f97d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf025a fdcde237-80a1-408d-b137-6275ab52f97d /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508581072932,"admin-job-type":0} /tidb/cdc/default/default/task/position/fdcde237-80a1-408d-b137-6275ab52f97d/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/05/31 16:59:45.112 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 16:59:45.173 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 16:59:45.229 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:45.431 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:45.441 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:45.526 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 16:59:45.528 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 16:59:45.529 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 16:59:45.533 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 2-th time, retry later [2024/05/31 16:59:45.543 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:45.640 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:45.645 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 7-th time, retry later [2024/05/31 16:59:45.877 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 16:59:45.901 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 16:59:45.725 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:45.725 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:45.739 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:45.752 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:45.825 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:45.846 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:45.850 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:45.929 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:45.945 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816d140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25935, start at 2024-05-31 16:59:43.842171137 +0800 CST m=+4.143900015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.813 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.813 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log check diff failed 2-th time, retry later [2024/05/31 16:59:46.093 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 16:59:46.093 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 16:59:46.105 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 16:59:46.106 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 16:59:46.037 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:46.140 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:46.414 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 16:59:46.437 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 16:59:46.249 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:46.325 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:46.526 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:46.630 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:46.727 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:46.729 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:46.745 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:46.746 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:46.843 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:46.845 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:46.852 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:46.852 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:46.856 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:46.944 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:46.955 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:46.804 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 16:59:46.806 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 16:59:46.812 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 16:59:46.816 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later [2024/05/31 16:59:47.050 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:47.235 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:47.250 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:47.343 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:47.428 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032340003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19240, start at 2024-05-31 16:59:23.663696097 +0800 CST m=+4.391989589 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.672 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 8-th time, retry later [2024/05/31 16:59:47.530 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:47.542 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:47.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:47.641 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:47.831 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:47.834 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:47.925 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:47.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:47.927 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:47.953 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] check diff failed 3-th time, retry later [2024/05/31 16:59:48.028 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:48.043 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:48.158 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:48.332 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:48.427 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:48.429 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:48.440 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] Starting CDC state checker... [2024/05/31 16:59:48.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:48.639 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:48.733 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table changefeed_pause_resume.t3 exists [2024/05/31 16:59:48.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:48.928 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:48.942 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:48.946 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 16:59:49.040 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:49.044 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:49.056 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:49.131 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:49.235 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:49.334 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:49.352 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:49.448 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:49.454 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:49.534 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:49.561 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:49.738 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' check diff failed 4-th time, retry later [2024/05/31 16:59:49.748 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:49.832 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:49.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:49.936 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 9-th time, retry later [2024/05/31 16:59:50.030 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:50.130 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:50.241 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:50.245 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:50.264 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] check diff failed 1-th time, retry later [2024/05/31 16:59:50.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:50.364 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:50.365 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:50.447 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:50.455 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] Verifying downstream PD is started... [2024/05/31 16:59:50.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:50.640 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:50.930 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:50.935 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:51.032 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:51.132 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1cfc2d60_6dd7_468f_bb3e_4ee46a81d9c0"] [2024/05/31 16:59:51.135 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:51.142 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:51.150 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 16:59:51.244 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:51.249 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:51.329 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 16:59:51.359 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:51.537 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs56ee64e1_beef_48f8_a6f9_7a884448e5ad"] check diff failed 5-th time, retry later [2024/05/31 16:59:51.542 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 16:59:51.562 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6d3c0107_31d5_4896_b442_632e1726119c"] [2024/05/31 16:59:51.639 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:51.642 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:51.649 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 16:59:51.742 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 16:59:51.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:51.834 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 16:59:52.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] check diff failed 10-th time, retry later [2024/05/31 16:59:52.327 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:52.350 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 16:59:52.355 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:52.429 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:52.441 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:52.444 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:52.456 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:52.463 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 16:59:52.539 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:52.548 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later [2024/05/31 16:59:52.626 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 16:59:53.147 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:53.250 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3c10a5cf_69ba_47f9_b79f_cafb0719c869"] [2024/05/31 16:59:53.257 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:53.329 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:53.334 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:53.346 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:53.352 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 16:59:53.352 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:53.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 16:59:53.360 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa0e80c15_629e_4f9e_88d5_79d9b977a7f6"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 11-th time, retry later [2024/05/31 16:59:53.740 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:54.059 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:54.066 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:54.067 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:54.149 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:54.149 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 16:59:54 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [2024/05/31 16:59:54.231 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:54.235 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 16:59:54.237 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:54.239 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 16:59:54.247 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:54.270 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:54.439 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later [2024/05/31 16:59:54.729 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:54.825 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc73bd650_ce0c_42bb_9a9b_08e1e8b41408"] [2024/05/31 16:59:54.928 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:54.928 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:54.932 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:54.933 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:55.037 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs71e2598e_0636_40ca_8089_532e375aea92"] [2024/05/31 16:59:55.133 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:55.138 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:55.149 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:55.150 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (25489) - No such process wait process 25489 exit for 1-th time... process 25489 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 16:59:53 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2568925691.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:55.259 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:55.455 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:55.543 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:55.649 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:55.651 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:55.659 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:55.661 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:55.731 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:55.755 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 16:59:55.929 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c88001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18369, start at 2024-05-31 16:59:30.515258692 +0800 CST m=+5.275935862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/31 16:59:56.032 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:56.037 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:56.043 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:56.133 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:56.233 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass check diff failed 12-th time, retry later [2024/05/31 16:59:56.434 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:56.449 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 16:59:56.637 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:56.638 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:56.655 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 16:59:56.735 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:56.736 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:56.744 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:56.765 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:56.766 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 16:59:55.182 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m4.370673875s"] table mark.finish_mark_0 exists table mark.finish_mark_1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_retryable_error [Fri May 31 16:59:56 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4154741549.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 16:59:56.842 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:56.850 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:56.926 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:56.933 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:57.244 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:57.249 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc004a700d8 TaskPositions:map[20a18345-9c0c-40bc-ac0d-052392370923:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing tableID: 84, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20a18345-9c0c-40bc-ac0d-052392370923 {"id":"20a18345-9c0c-40bc-ac0d-052392370923","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf028d 20a18345-9c0c-40bc-ac0d-052392370923 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508607287343,"admin-job-type":0} /tidb/cdc/default/default/task/position/20a18345-9c0c-40bc-ac0d-052392370923/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc004a700d8 TaskPositions:map[20a18345-9c0c-40bc-ac0d-052392370923:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing tableID: 84, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20a18345-9c0c-40bc-ac0d-052392370923 {"id":"20a18345-9c0c-40bc-ac0d-052392370923","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf028d 20a18345-9c0c-40bc-ac0d-052392370923 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508607287343,"admin-job-type":0} /tidb/cdc/default/default/task/position/20a18345-9c0c-40bc-ac0d-052392370923/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {ClusterID:default ID:default/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 Info:{"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} Status:0xc004a700d8 TaskPositions:map[20a18345-9c0c-40bc-ac0d-052392370923:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing tableID: 84, resolvedTs: 450139508581072932, checkpointTs: 450139508581072932, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20a18345-9c0c-40bc-ac0d-052392370923 {"id":"20a18345-9c0c-40bc-ac0d-052392370923","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddebf028d 20a18345-9c0c-40bc-ac0d-052392370923 /tidb/cdc/default/default/changefeed/info/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"upstream-id":7375084744008070100,"namespace":"default","changefeed-id":"ef6b70ac-3b8e-443a-8910-e22cc2ff76f4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T16:58:11.790585206+08:00","start-ts":450139492629610502,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139492656087042} /tidb/cdc/default/default/changefeed/status/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":450139508581072932,"min-table-barrier-ts":450139508607287343,"admin-job-type":0} /tidb/cdc/default/default/task/position/20a18345-9c0c-40bc-ac0d-052392370923/ef6b70ac-3b8e-443a-8910-e22cc2ff76f4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375084744008070100 {"id":7375084744008070100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 2-th check, retry later [2024/05/31 16:59:57.446 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:57.450 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 16:59:57.544 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 16:59:57.557 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:57.629 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:57.633 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:57.653 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:57.726 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 16:59:57.748 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 16:59:57.749 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:57.758 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 16:59:57.827 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:57.961 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:58.026 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:58.043 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs813df1d3_2414_4db2_9198_bdc4af5bf253"] [2024/05/31 16:59:58.149 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:58.231 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 13-th time, retry later check diff failed 5-th time, retry later ***************** properties ***************** "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "requestdistribution"="uniform" "scanproportion"="0" "workload"="core" "recordcount"="5000" "updateproportion"="0" "insertproportion"="0" "mysql.db"="consistent_replicate_storage_s3" "readproportion"="0" "dotransactions"="false" "fieldcount"="100" "readallfields"="true" "threadcount"="10" "mysql.port"="4000" ********************************************** [2024/05/31 16:59:58.427 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 16:59:58.534 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:58.535 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:58.547 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:58.551 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 16:59:58.651 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 16:59:58.653 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:58.735 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 16:59:58.742 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 16:59:58.745 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table kill_owner_with_ddl.t3 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 3-th check, retry later check diff failed 1-th time, retry later [2024/05/31 16:59:59.445 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:59.445 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:59.541 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 16:59:59.541 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 16:59:59.638 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 16:59:59.642 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 16:59:59.739 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 16:59:59.927 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 16:59:59.929 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 16:59:59.936 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 16:59:59.938 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:00.029 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:00.045 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:00.061 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:00.138 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:00.139 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff failed 14-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 08:59:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013c9650 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139520874053638, checkpointTs: 450139520874053638, state: Replicating tableID: 82, resolvedTs: 450139520887160833, checkpointTs: 450139520887160833, state: Replicating tableID: 84, resolvedTs: 450139520900005890, checkpointTs: 450139520900005890, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139520874053638,"min-table-barrier-ts":450139520900005890,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013c9650 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139520874053638, checkpointTs: 450139520874053638, state: Replicating tableID: 82, resolvedTs: 450139520887160833, checkpointTs: 450139520887160833, state: Replicating tableID: 84, resolvedTs: 450139520900005890, checkpointTs: 450139520900005890, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-fi+ grep -q 'failed to get info:' le-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139520874053638,"min-table-barrier-ts":450139520900005890,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013c9650 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139520874053638, checkpointTs: 450139520874053638, state: Replicating tableID: 82, resolvedTs: 450139520887160833, checkpointTs: 450139520887160833, state: Replicating tableID: 84, resolvedTs: 450139520900005890, checkpointTs: 450139520900005890, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139520874053638,"min-table-barrier-ts":450139520900005890,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully [2024/05/31 17:00:00.254 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:00.254 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:00.331 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:00:00.338 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:00:00.355 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:00.427 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:00.449 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] owner pid: 41552 owner id 3c7e822f-6642-4877-aa28-c2236147ac29 [Fri May 31 17:00:00 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4169241694.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' [2024/05/31 17:00:00.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:00:00.743 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:00.745 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:00.745 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:00.834 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:00.850 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:00.943 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:00.952 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37826468000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18813, start at 2024-05-31 16:59:59.660910436 +0800 CST m=+5.002147018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log [2024/05/31 17:00:01.027 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:01.127 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:01.131 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:01.140 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:00:01.154 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:01.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:01.229 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:01.251 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:01.530 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:01.542 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:01.552 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:01.553 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:01.555 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:01.649 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:01.736 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table mark.finish_mark_1 not exists for 4-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:00:01 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> [2024/05/31 17:00:01.826 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:01.827 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:01.855 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:01.927 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:01.937 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:01.951 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:01.956 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:01.957 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:00:02.046 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 1-th time, retry later [2024/05/31 17:00:02.430 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] check diff failed 15-th time, retry later [2024/05/31 17:00:02.532 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:02.534 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:02.552 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:02.627 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:02.725 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:00:02.755 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:02.850 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:02.925 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:02.926 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:02.932 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:02.940 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting CDC state checker... [2024/05/31 17:00:03.040 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:03.041 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:03.126 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:03.127 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:03.327 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:03.344 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:03.358 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:03.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:03.534 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:03.554 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:03.642 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:03.654 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:03.654 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:03.729 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:03.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:03.761 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] Run finished, takes 5.726515953s INSERT - Takes(s): 5.7, Count: 5000, OPS: 874.8, Avg(us): 10827, Min(us): 1746, Max(us): 100877, 95th(us): 83000, 99th(us): 90000 table mark.finish_mark_1 not exists for 5-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 17:00:03.845 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:03.854 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:04.037 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:04.043 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc003411218 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 971c06ec-87c8-49c9-ba4a-6191e55c8d7e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/971c06ec-87c8-49c9-ba4a-6191e55c8d7e {"id":"971c06ec-87c8-49c9-ba4a-6191e55c8d7e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599d3 971c06ec-87c8-49c9-ba4a-6191e55c8d7e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521673592833,"min-table-barrier-ts":450139521686700035,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/971c06ec-87c8-49c9-ba4a-6191e55c8d7e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc003411218 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 971c06ec-87c8-49c9-ba4a-6191e55c8d7e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/971c06ec-87c8-49c9-ba4a-6191e55c8d7e {"id":"971c06ec-87c8-49c9-ba4a-6191e55c8d7e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599d3 971c06ec-87c8-49c9-ba4a-6191e55c8d7e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521673592833,"min-table-barrier-ts":450139521686700035,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/971c06ec-87c8-49c9-ba4a-6191e55c8d7e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc003411218 TaskPositions:map[3c7e822f-6642-4877-aa28-c2236147ac29:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 971c06ec-87c8-49c9-ba4a-6191e55c8d7e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c7e822f-6642-4877-aa28-c2236147ac29 {"id":"3c7e822f-6642-4877-aa28-c2236147ac29","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/971c06ec-87c8-49c9-ba4a-6191e55c8d7e {"id":"971c06ec-87c8-49c9-ba4a-6191e55c8d7e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599b0 3c7e822f-6642-4877-aa28-c2236147ac29 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599d3 971c06ec-87c8-49c9-ba4a-6191e55c8d7e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521673592833,"min-table-barrier-ts":450139521686700035,"admin-job-type":0} /tidb/cdc/default/default/task/position/3c7e822f-6642-4877-aa28-c2236147ac29/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/971c06ec-87c8-49c9-ba4a-6191e55c8d7e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "3c7e822f-6642-4877-aa28-c2236147ac29" | grep id "cluster-id": "default" "id": "971c06ec-87c8-49c9-ba4a-6191e55c8d7e", "cluster-id": "default" run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= capture_id: 971c06ec-87c8-49c9-ba4a-6191e55c8d7e [2024/05/31 17:00:04.138 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] % Total % Received % Xferd Average Speed Time Time Time Current The 1 times to try to start tidb cluster... Release Version: 6.5.9 Edition: Community Dload Upload Total Spent Left Speed [2024/05/31 17:00:04.235 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 32407 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" [2024/05/31 17:00:04.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] UTC Build Time: 2024-05-10 07:21:43 }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '3c7e822f-6642-4877-aa28-c2236147ac29' 3c7e822f-6642-4877-aa28-c2236147ac29 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:00:04.337 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:04.355 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:04.357 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:04.357 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:04.434 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:04.450 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:04.458 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 17:00:04.540 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:04.628 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:04.734 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:04.859 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Fri May 31 17:00:04 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4185041852.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:00:04 CST 2024] <<<<<< run test case sink_hang success! >>>>>> check diff failed 2-th time, retry later [2024/05/31 17:00:04.937 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:04.952 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:04.972 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:05.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:05.034 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:05.061 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:05.077 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:05.149 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:05.260 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:05.446 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:05.536 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:05.547 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:05.629 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 17:00:05.631 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:05.643 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:05.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:05.662 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table mark.finish_mark_1 not exists for 6-th check, retry later [2024/05/31 17:00:05.739 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:05.853 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:00:06.055 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 17:00:06.147 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later [2024/05/31 17:00:06.226 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:06.248 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 17:00:06.252 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:06.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:06.335 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 17:00:06.347 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/31 17:00:06.364 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032580016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19168, start at 2024-05-31 16:59:23.714783517 +0800 CST m=+4.512773817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378032340003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:19240, start at 2024-05-31 16:59:23.663696097 +0800 CST m=+4.391989589 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:23.672 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:23.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:23.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:00:06 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2305523057.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:00:06.525 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 3-th time, retry later [2024/05/31 17:00:06.742 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 17:00:06.748 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsaf755bdd_fd8a_47c9_ae9c_8c6e9af0b157"] [2024/05/31 17:00:06.758 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs561442a1_3ef1_42df_85ff_d523ffb4fb9d"] [2024/05/31 17:00:06.826 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:06.841 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:06.931 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/31 17:00:06.936 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 450139511436345353 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Fri May 31 16:59:54 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Fri May 31 16:59:54 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully [Fri May 31 17:00:06 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> [2024/05/31 17:00:06.956 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:07.035 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/31 17:00:07.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:07.438 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table mark.finish_mark_1 not exists for 7-th check, retry later [2024/05/31 17:00:07.451 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:00:07.454 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:00:07.526 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:00:07.531 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 17:00:07.548 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/31 17:00:07.559 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 17:00:07.573 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:07.641 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/31 17:00:07.672 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:07.953 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/31 17:00:08.029 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:00:08.054 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:00:08.076 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:00:08.141 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:08.143 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/31 17:00:08.159 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816d140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25935, start at 2024-05-31 16:59:43.842171137 +0800 CST m=+4.143900015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.813 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.813 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 17:00:08.251 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:08.450 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 4-th time, retry later [2024/05/31 17:00:08.529 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs48cc979d_194b_4006_a502_cf5d465e6d64"] [2024/05/31 17:00:08.538 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:00:08.565 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:08.635 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:08.644 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later [2024/05/31 17:00:08.733 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/31 17:00:08.940 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:09.028 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3713bd1f_638b_4706_a1b8_0badfb6595db"] [2024/05/31 17:00:09.033 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:00:09.034 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:09.140 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:00:09.166 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:00:09.230 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table mark.finish_mark_1 not exists for 8-th check, retry later [2024/05/31 17:00:09.535 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:00:09.635 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:00:09.639 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:00:09.644 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:00:09.652 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:00:09.692 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:09.748 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:00:09.752 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:00:09 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2310923111.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:00:09.870 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:10.060 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:00:10.064 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:00:10.071 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:10.131 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:00:10.235 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:10.258 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:10.260 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:10.361 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:10.567 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later [2024/05/31 17:00:10.628 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:10.635 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:10.647 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:10.697 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:10.738 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8e8c1ed8_b3e2_4783_8d88_6973d4101cd6"] [2024/05/31 17:00:10.750 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:00:10.753 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:10.777 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:11.062 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:11.063 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:00:11.128 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:11.148 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:00:11.240 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:11.248 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:00:11.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:00:11.266 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:11.327 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:11.334 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table mark.finish_mark_1 not exists for 9-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001530b40 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521935474689,"min-table-barrier-ts":450139521948844035,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001530b40 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521935474689,"min-table-barrier-ts":450139521948844035,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001530b40 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139521935474689,"min-table-barrier-ts":450139521948844035,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/31 17:00:11.557 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:00:11.560 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] "is-owner": true, run task successfully [2024/05/31 17:00:11.647 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:11.657 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:11.755 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:11.765 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:00:11.834 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... owner pid: 41855 owner id 91ebbf2a-b676-40b1-93df-8652263b450e [Fri May 31 17:00:11 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4199341995.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:00:11.863 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:11.929 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:11.933 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:12.229 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:12.237 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:12.263 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:12.355 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:12.444 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:00:12.448 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:12.460 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:00:12.547 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/31 17:00:12.547 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:12.552 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later [2024/05/31 17:00:12.848 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:12.859 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:12 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b188ac54-3eca-4e2f-8042-609bbb2de560 {"id":"b188ac54-3eca-4e2f-8042-609bbb2de560","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8eb1 b188ac54-3eca-4e2f-8042-609bbb2de560 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b188ac54-3eca-4e2f-8042-609bbb2de560 {"id":"b188ac54-3eca-4e2f-8042-609bbb2de560","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8eb1 b188ac54-3eca-4e2f-8042-609bbb2de560 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b188ac54-3eca-4e2f-8042-609bbb2de560 {"id":"b188ac54-3eca-4e2f-8042-609bbb2de560","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cf1cd9a7-91d4-459b-be66-cf8d78747059 {"id":"cf1cd9a7-91d4-459b-be66-cf8d78747059","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8e8e cf1cd9a7-91d4-459b-be66-cf8d78747059 /tidb/cdc/default/__cdc_meta__/owner/22318fcde08d8eb1 b188ac54-3eca-4e2f-8042-609bbb2de560 /tidb/cdc/default/default/upstream/7375085726924015780 {"id":7375085726924015780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/31 17:00:12.948 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] check diff successfully [2024/05/31 17:00:13.125 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:13.137 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:00:13.159 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:13.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:13.235 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:00:13.261 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:13.263 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore [2024/05/31 17:00:13.429 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:13.449 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:13.541 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:00:13.666 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:00:13.725 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:00:13.751 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:13.826 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:00:13.828 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/31 17:00:13.853 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:13.856 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_1 not exists for 10-th check, retry later [2024/05/31 17:00:13.934 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:00:13.950 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:14.082 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:00:14.126 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + workdir=/tmp/tidb_cdc_test/canal_json_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_basic ++ date + echo '[Fri May 31 17:00:12 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>>' [Fri May 31 17:00:12 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml [2024/05/31 17:00:14.176 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:00:14.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:14.275 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/31 17:00:14.293 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later [2024/05/31 17:00:14.426 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:00:14.446 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:00:14.492 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:00:14.557 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:14.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:00:14.764 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:00:14.780 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/31 17:00:14.826 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff failed 1-th time, retry later [2024/05/31 17:00:14.929 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/31 17:00:15.130 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/31 17:00:15.132 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037d23a8 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} a4fd161d-f515-42d8-993b-47df82c90622:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a4fd161d-f515-42d8-993b-47df82c90622 {"id":"a4fd161d-f515-42d8-993b-47df82c90622","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59a25 a4fd161d-f515-42d8-993b-47df82c90622 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139524819320835,"min-table-barrier-ts":450139524832165893,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4fd161d-f515-42d8-993b-47df82c90622/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037d23a8 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} a4fd161d-f515-42d8-993b-47df82c90622:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a4fd161d-f515-42d8-993b-47df82c90622 {"id":"a4fd161d-f515-42d8-993b-47df82c90622","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59a25 a4fd161d-f515-42d8-993b-47df82c90622 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139524819320835,"min-table-barrier-ts":450139524832165893,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4fd161d-f515-42d8-993b-47df82c90622/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037d23a8 TaskPositions:map[91ebbf2a-b676-40b1-93df-8652263b450e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} a4fd161d-f515-42d8-993b-47df82c90622:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91ebbf2a-b676-40b1-93df-8652263b450e {"id":"91ebbf2a-b676-40b1-93df-8652263b450e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a4fd161d-f515-42d8-993b-47df82c90622 {"id":"a4fd161d-f515-42d8-993b-47df82c90622","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec599f7 91ebbf2a-b676-40b1-93df-8652263b450e /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59a25 a4fd161d-f515-42d8-993b-47df82c90622 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139524819320835,"min-table-barrier-ts":450139524832165893,"admin-job-type":0} /tidb/cdc/default/default/task/position/91ebbf2a-b676-40b1-93df-8652263b450e/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4fd161d-f515-42d8-993b-47df82c90622/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "91ebbf2a-b676-40b1-93df-8652263b450e" | grep id table test.simple1 not exists for 1-th check, retry later table test.simple1 not exists for 2-th check, retry later table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (30115) - No such process wait process 30115 exit for 1-th time... process 30115 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep e9e9ea5d-d031-4605-91cb-116c27e1075e -A1 | grep '"is-owner": true' [2024/05/31 17:00:15.171 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:00:15.263 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] "cluster-id": "default" "id": "a4fd161d-f515-42d8-993b-47df82c90622", "cluster-id": "default" run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully [Fri May 31 17:00:15 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3064230644.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:00:15.482 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/31 17:00:15.604 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18284, start at 2024-05-31 16:59:30.483281299 +0800 CST m=+5.337217148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.488 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.488 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37809c88001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:18369, start at 2024-05-31 16:59:30.515258692 +0800 CST m=+5.275935862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:30.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:30.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:30.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:00:15 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.2223822240.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 capture_id: a4fd161d-f515-42d8-993b-47df82c90622 table mark.finish_mark_1 not exists for 11-th check, retry later [2024/05/31 17:00:16.218 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m13.566456819s"] table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... [2024/05/31 17:00:16.730 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m14.077934408s"] table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later check diff failed 2-th time, retry later Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 12-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 3-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} Status:0xc001b9a540 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} Status:0xc001b9a4e0 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43895994-b450-4664-8111-8ef97e4e82c1 {"id":"43895994-b450-4664-8111-8ef97e4e82c1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1bb1d 43895994-b450-4664-8111-8ef97e4e82c1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.776644386+08:00","start-ts":450139510203482122,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139524713414667} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450139522642214913,"min-table-barrier-ts":450139524437901315,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} Status:0xc001b9a540 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} Status:0xc001b9a4e0 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43895994-b450-4664-8111-8ef97e4e82c1 {"id":"43895994-b450-4664-8111-8ef97e4e82c1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1bb1d 43895994-b450-4664-8111-8ef97e4e82c1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.776644386+08:00","start-ts":450139510203482122,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139524713414667} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450139522642214913,"min-table-barrier-ts":450139524437901315,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} Status:0xc001b9a540 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} Status:0xc001b9a4e0 TaskPositions:map[43895994-b450-4664-8111-8ef97e4e82c1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e9e9ea5d-d031-4605-91cb-116c27e1075e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43895994-b450-4664-8111-8ef97e4e82c1 {"id":"43895994-b450-4664-8111-8ef97e4e82c1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e9e9ea5d-d031-4605-91cb-116c27e1075e {"id":"e9e9ea5d-d031-4605-91cb-116c27e1075e","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1ba56 e9e9ea5d-d031-4605-91cb-116c27e1075e /tidb/cdc/default/__cdc_meta__/owner/22318fcddfb1bb1d 43895994-b450-4664-8111-8ef97e4e82c1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:18.657913933+08:00","start-ts":450139510164160521,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510177267721} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.776644386+08:00","start-ts":450139510203482122,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139524713414667} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7375085480949866620,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-31T16:59:18.861880745+08:00","start-ts":450139510229696523,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139510229696525} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":450139522642214913,"min-table-barrier-ts":450139524437901315,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":450139525722406923,"min-table-barrier-ts":450139525722406923,"admin-job-type":0} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/43895994-b450-4664-8111-8ef97e4e82c1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e9e9ea5d-d031-4605-91cb-116c27e1075e/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085480949866620 {"id":7375085480949866620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' "address": "127.0.0.1:8300", run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/default/upstream/7375085753114098292 {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/default/upstream/7375085753114098292 {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/default/upstream/7375085753114098292 {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.22344.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release pass test: list changefeed Create changefeed successfully! ID: 15502b00-b9c0-41e8-b97c-a028c6f427b3 Info: {"upstream_id":7375085753114098292,"namespace":"default","id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:00:19.004218152+08:00","start_ts":450139525985337347,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139525985337347,"checkpoint_ts":450139525985337347,"checkpoint_time":"2024-05-31 17:00:18.926"} PASS table mark.finish_mark_2 not exists for 2-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later pass test: get changefeed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378386200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33648, start at 2024-05-31 17:00:18.207062198 +0800 CST m=+4.741619849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log pass test: resume changefeed Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later check diff failed 1-th time, retry later pass test: pause changefeed pass test: rebalance table Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later pass test: move table table mark.finish_mark_2 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 2-th check, retry later pass test: get processors Verifying downstream PD is started... pass test: list processors pass test: set log level ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later pass test: remove changefeed pass test: resign owner nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_2 not exists for 4-th check, retry later check diff failed 5-th time, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37826468000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18813, start at 2024-05-31 16:59:59.660910436 +0800 CST m=+5.002147018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_2 not exists for 3-th check, retry later Starting CDC state checker... table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_2 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 6-th time, retry later check diff failed 3-th time, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_2 not exists for 4-th check, retry later table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff successfully check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816c74001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25838, start at 2024-05-31 16:59:43.809402559 +0800 CST m=+4.159860040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37816d140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:25935, start at 2024-05-31 16:59:43.842171137 +0800 CST m=+4.143900015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:43.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:43.813 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:43.813 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:00:27 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.3002730029.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840d54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37695, start at 2024-05-31 17:00:26.847710995 +0800 CST m=+4.348124706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.837 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.837 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_2 not exists for 7-th check, retry later check diff failed 5-th time, retry later table mark.finish_mark_2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later check diff failed 1-th time, retry later table mark.finish_mark_2 not exists for 8-th check, retry later Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19eb6e94-2303-46c4-8e11-c2aa6d1921e0 {"id":"19eb6e94-2303-46c4-8e11-c2aa6d1921e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0debf90 19eb6e94-2303-46c4-8e11-c2aa6d1921e0 /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19eb6e94-2303-46c4-8e11-c2aa6d1921e0 {"id":"19eb6e94-2303-46c4-8e11-c2aa6d1921e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0debf90 19eb6e94-2303-46c4-8e11-c2aa6d1921e0 /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19eb6e94-2303-46c4-8e11-c2aa6d1921e0 {"id":"19eb6e94-2303-46c4-8e11-c2aa6d1921e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0debf90 19eb6e94-2303-46c4-8e11-c2aa6d1921e0 /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.30095.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7375085811935213959,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-31T17:00:31.166648393+08:00","start_ts":450139529159376902,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139529159376902,"checkpoint_ts":450139529159376902,"checkpoint_time":"2024-05-31 17:00:31.034"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table mark.finish_mark_2 not exists for 7-th check, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378451540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26839, start at 2024-05-31 17:00:31.195410664 +0800 CST m=+10.933354119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.212 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.189 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.189 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later check diff failed 2-th time, retry later table mark.finish_mark_2 not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Fri May 31 17:00:33 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later check diff failed 3-th time, retry later table mark.finish_mark_2 not exists for 10-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 2-th check, retry later table consistent_replicate_storage_s3.t2 exists check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 11-th check, retry later check diff successfully empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:00:37 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3598835990.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case delete_owner_key [Fri May 31 17:00:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4238942391.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... check diff failed 5-th time, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later skip some SQLs in tidb v4.0.x table common_1.v1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 4-th check, retry later table common_1.v1 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/d64634d6-2394-41ab-a49b-8bf0a4351a8d {ClusterID:default ID:default/d64634d6-2394-41ab-a49b-8bf0a4351a8d Info:{"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} Status:0xc000cf8eb8 TaskPositions:map[e3db9427-3bba-42d0-ac22-a14038ebdb31:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 89, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 94, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 91, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 92, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 82, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 84, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 87, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3db9427-3bba-42d0-ac22-a14038ebdb31 {"id":"e3db9427-3bba-42d0-ac22-a14038ebdb31","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ec7e e3db9427-3bba-42d0-ac22-a14038ebdb31 /tidb/cdc/default/default/changefeed/info/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} /tidb/cdc/default/default/changefeed/status/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":450139522472345603,"min-table-barrier-ts":450139531516575746,"admin-job-type":0} /tidb/cdc/default/default/task/position/e3db9427-3bba-42d0-ac22-a14038ebdb31/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d64634d6-2394-41ab-a49b-8bf0a4351a8d {ClusterID:default ID:default/d64634d6-2394-41ab-a49b-8bf0a4351a8d Info:{"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} Status:0xc000cf8eb8 TaskPositions:map[e3db9427-3bba-42d0-ac22-a14038ebdb31:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 89, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 94, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 91, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 92, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 82, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 84, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 87, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3db9427-3bba-42d0-ac22-a14038ebdb31 {"id":"e3db9427-3bba-42d0-ac22-a14038ebdb31","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ec7e e3db9427-3bba-42d0-ac22-a14038ebdb31 /tidb/cdc/default/default/changefeed/info/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} /tidb/cdc/default/default/changefeed/status/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":450139522472345603,"min-table-barrier-ts":450139531516575746,"admin-job-type":0} /tidb/cdc/default/default/task/position/e3db9427-3bba-42d0-ac22-a14038ebdb31/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d64634d6-2394-41ab-a49b-8bf0a4351a8d {ClusterID:default ID:default/d64634d6-2394-41ab-a49b-8bf0a4351a8d Info:{"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} Status:0xc000cf8eb8 TaskPositions:map[e3db9427-3bba-42d0-ac22-a14038ebdb31:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 88, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 89, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 94, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 91, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 92, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 82, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 84, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating tableID: 87, resolvedTs: 450139522472345603, checkpointTs: 450139522472345603, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3db9427-3bba-42d0-ac22-a14038ebdb31 {"id":"e3db9427-3bba-42d0-ac22-a14038ebdb31","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcddf80ec7e e3db9427-3bba-42d0-ac22-a14038ebdb31 /tidb/cdc/default/default/changefeed/info/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"upstream-id":7375085424323535755,"namespace":"default","changefeed-id":"d64634d6-2394-41ab-a49b-8bf0a4351a8d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:02.015931617+08:00","start-ts":450139505787142147,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num"+ grep -q 'etcd info' :16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139505813356550} /tidb/cdc/default/default/changefeed/status/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":450139522472345603,"min-table-barrier-ts":450139531516575746,"admin-job-type":0} /tidb/cdc/default/default/task/position/e3db9427-3bba-42d0-ac22-a14038ebdb31/d64634d6-2394-41ab-a49b-8bf0a4351a8d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085424323535755 {"id":7375085424323535755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully table mark.finish_mark_3 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0012366c0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 84, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 80, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139531635064839,"min-table-barrier-ts":450139531661279233,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0012366c0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 84, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 80, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139531635064839,"min-table-barrier-ts":450139531661279233,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 + grep -q 'failed to get info:' {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0012366c0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 84, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating tableID: 80, resolvedTs: 450139531635064839, checkpointTs: 450139531635064839, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139531635064839,"min-table-barrier-ts":450139531661279233,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully table mark.finish_mark_3 not exists for 5-th check, retry later owner pid: 42394 owner id 1974041b-df29-4a1a-bbb3-665d636d6e82 default owner key /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae [Fri May 31 17:00:41 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4254342545.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d2500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22204, start at 2024-05-31 17:00:39.492835094 +0800 CST m=+5.839851040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log check diff failed 1-th time, retry later table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table mark.finish_mark_3 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378264400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18749, start at 2024-05-31 16:59:59.673308584 +0800 CST m=+5.093838726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37826468000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:18813, start at 2024-05-31 16:59:59.660910436 +0800 CST m=+5.002147018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:01:59.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-16:59:59.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:49:59.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.22713.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139532321095681 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139532321095681 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table mark.finish_mark_3 not exists for 6-th check, retry later table common_1.recover_and_insert not exists for 2-th check, retry later Starting CDC state checker... table mark.finish_mark_3 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:00:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000840ea0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 4b7d4d47-cd67-4a55-a56b-451b50cb9d56:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4b7d4d47-cd67-4a55-a56b-451b50cb9d56 {"id":"4b7d4d47-cd67-4a55-a56b-451b50cb9d56","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ad5 4b7d4d47-cd67-4a55-a56b-451b50cb9d56 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139532421496844,"min-table-barrier-ts":450139532709855234,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4b7d4d47-cd67-4a55-a56b-451b50cb9d56/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000840ea0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 4b7d4d47-cd67-4a55-a56b-451b50cb9d56:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4b7d4d47-cd67-4a55-a56b-451b50cb9d56 {"id":"4b7d4d47-cd67-4a55-a56b-451b50cb9d56","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ad5 4b7d4d47-cd67-4a55-a56b-451b50cb9d56 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139532421496844,"min-table-barrier-ts":450139532709855234,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4b7d4d47-cd67-4a55-a56b-451b50cb9d56/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc000840ea0 TaskPositions:map[1974041b-df29-4a1a-bbb3-665d636d6e82:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 4b7d4d47-cd67-4a55-a56b-451b50cb9d56:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1974041b-df29-4a1a-bbb3-665d636d6e82 {"id":"1974041b-df29-4a1a-bbb3-665d636d6e82","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4b7d4d47-cd67-4a55-a56b-451b50cb9d56 {"id":"4b7d4d47-cd67-4a55-a56b-451b50cb9d56","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59aae 1974041b-df29-4a1a-bbb3-665d636d6e82 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ad5 4b7d4d47-cd67-4a55-a56b-451b50cb9d56 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139532421496844,"min-table-barrier-ts":450139532709855234,"admin-job-type":0} /tidb/cdc/default/default/task/position/1974041b-df29-4a1a-bbb3-665d636d6e82/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4b7d4d47-cd67-4a55-a56b-451b50cb9d56/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "1974041b-df29-4a1a-bbb3-665d636d6e82 default" | grep id "id": "4b7d4d47-cd67-4a55-a56b-451b50cb9d56", run task successfully check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378386200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33648, start at 2024-05-31 17:00:18.207062198 +0800 CST m=+4.741619849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 7-th check, retry later capture_id: 4b7d4d47-cd67-4a55-a56b-451b50cb9d56 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '4b7d4d47-cd67-4a55-a56b-451b50cb9d56' 4b7d4d47-cd67-4a55-a56b-451b50cb9d56 run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 177 100 177 0 0 149k 0 --:--:-- --:--:-- --:--:-- 172k "is_owner": false run task successfully start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table common_1.recover_and_insert not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 5-th check, retry later check diff failed 3-th time, retry later table mark.finish_mark_3 exists table mark.finish_mark_4 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 table mark.finish_mark_4 not exists for 2-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table common_1.finish_mark not exists for 2-th check, retry later check diff failed 5-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840d54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37695, start at 2024-05-31 17:00:26.847710995 +0800 CST m=+4.348124706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.837 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.837 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_4 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table common_1.finish_mark not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 8-th check, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378451540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26839, start at 2024-05-31 17:00:31.195410664 +0800 CST m=+10.933354119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.212 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.189 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.189 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_4 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table common_1.finish_mark exists check diff successfully pass test: get tso cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:00:53 CST 2024] <<<<<< run test case http_api success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:00:54 CST 2024] <<<<<< run test case common_1 success! >>>>>> nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 10-th check, retry later table mark.finish_mark_4 not exists for 6-th check, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later table mark.finish_mark_3 not exists for 11-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785faf8001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32513, start at 2024-05-31 17:00:58.479167539 +0800 CST m=+4.310229782 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 4-th time, retry later table mark.finish_mark_3 not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts d64634d6-2394-41ab-a49b-8bf0a4351a8d 450139537493196801 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/d64634d6-2394-41ab-a49b-8bf0a4351a8d/meta check diff failed 5-th time, retry later Starting CDC state checker... global resolved ts 450139537047552002 not forward to 450139537493196801 run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) table mark.finish_mark_3 not exists for 13-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... table mark.finish_mark_4 not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3783864c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33551, start at 2024-05-31 17:00:18.226289281 +0800 CST m=+4.861339889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.195 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.195 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378386200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:33648, start at 2024-05-31 17:00:18.207062198 +0800 CST m=+4.741619849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:18.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:18.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:18.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:01:04 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.3809138093.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_redo_resolved_ts d64634d6-2394-41ab-a49b-8bf0a4351a8d 450139537493196801 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/d64634d6-2394-41ab-a49b-8bf0a4351a8d/meta run task successfully table mark.finish_mark_3 not exists for 14-th check, retry later check diff failed 6-th time, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/31 17:01:05.613 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 17:01:05.615 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:01:05.618 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 17:01:05.618 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139531817779211] [resolvedTs=450139537571840003] [2024/05/31 17:01:05.618 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:01:05.619 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:01:05.619 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:01:05.619 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:01:05.623 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.779615ms] [2024/05/31 17:01:05.624 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:01:05.628 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.629 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 17:01:05.630 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 17:01:05.630 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 17:01:05.630 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 17:01:05.636 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:01:05.640 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later [2024/05/31 17:01:06.184 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=566.144346ms] [2024/05/31 17:01:06.185 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":450139531817779206,\"CommitTs\":450139531817779211,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_s3\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:01:07.256 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139537571840003] [2024/05/31 17:01:07.256 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139537571840003] [2024/05/31 17:01:07.356 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139537571840003] [2024/05/31 17:01:07.356 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139537571840003] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:07 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 {"id":"c92d6fb8-6d0e-47a6-89d4-860d8f0285e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde16ede32 c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 /tidb/cdc/default/default/upstream/7375085957177525259 {"id":7375085957177525259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 {"id":"c92d6fb8-6d0e-47a6-89d4-860d8f0285e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde16ede32 c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 /tidb/cdc/default/default/upstream/7375085957177525259 {"id":7375085957177525259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 {"id":"c92d6fb8-6d0e-47a6-89d4-860d8f0285e0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde16ede32 c92d6fb8-6d0e-47a6-89d4-860d8f0285e0 /tidb/cdc/default/default/upstream/7375085957177525259 {"id":7375085957177525259,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.38171.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table mark.finish_mark_3 not exists for 15-th check, retry later Create changefeed successfully! ID: 5fd530e6-757d-43c1-90d8-80c8c79c9a33 Info: {"upstream_id":7375085957177525259,"namespace":"default","id":"5fd530e6-757d-43c1-90d8-80c8c79c9a33","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:01:07.556967794+08:00","start_ts":450139538700369925,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139538700369925,"checkpoint_ts":450139538700369925,"checkpoint_time":"2024-05-31 17:01:07.430"} PASS [2024/05/31 17:01:07.858 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450139537571840003] [2024/05/31 17:01:07.858 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=450139537571840003] [2024/05/31 17:01:07.958 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=450139537571840003] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 17:01:07.958 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] Apply redo log successfully coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table mark.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully [Fri May 31 17:01:08 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 16-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d2500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22204, start at 2024-05-31 17:00:39.492835094 +0800 CST m=+5.839851040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 exists empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass table mark.finish_mark_3 not exists for 17-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840bec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37591, start at 2024-05-31 17:00:26.763210383 +0800 CST m=+4.330302854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37840d54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:37695, start at 2024-05-31 17:00:26.847710995 +0800 CST m=+4.348124706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:26.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:26.837 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:26.837 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41768.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_kill_capture [Fri May 31 17:01:11 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4300743009.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark not exists for 4-th check, retry later + set +x + tso='450139539824967681 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139539824967681 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table mark.finish_mark_3 not exists for 18-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378455b80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26732, start at 2024-05-31 17:00:31.490816715 +0800 CST m=+11.331108419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378451540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--cmg2r, pid:26839, start at 2024-05-31 17:00:31.195410664 +0800 CST m=+10.933354119 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:31.212 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:31.189 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:31.189 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:01:13 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.3110731109.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 139 100 35 100 104 732 2176 --:--:-- --:--:-- --:--:-- 2212 { "status": true, "message": "" start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table mark.finish_mark_3 not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001f753c8 TaskPositions:map[d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 80, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 84, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139540312031237,"min-table-barrier-ts":450139540574175237,"admin-job-type":0} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001f753c8 TaskPositions:map[d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 80, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 84, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139540312031237,"min-table-barrier-ts":450139540574175237,"admin-job-type":0} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc001f753c8 TaskPositions:map[d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 80, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating tableID: 84, resolvedTs: 450139540561068037, checkpointTs: 450139540561068037, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-fi+ grep -q 'etcd info' le-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139540312031237,"min-table-barrier-ts":450139540574175237,"admin-job-type":0} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later owner pid: 43012 owner id d22039c1-63b5-404f-98c2-c01fc42d6ead table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark not exists for 6-th check, retry later table owner_resign.t1 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff failed 1-th time, retry later }lease 22318fcde08d8eb1 revoked check diff successfully < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e {"id":"dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde18dd58e dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e /tidb/cdc/default/default/upstream/7375085979494935060 {"id":7375085979494935060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e {"id":"dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde18dd58e dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e /tidb/cdc/default/default/upstream/7375085979494935060 {"id":7375085979494935060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e {"id":"dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde18dd58e dd0519c0-d90c-457d-81ae-9ef5b0e4bc1e /tidb/cdc/default/default/upstream/7375085979494935060 {"id":7375085979494935060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:01:16 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Fri May 31 17:01:17 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4318243184.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 20-th check, retry later table mark.finish_mark not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Fri May 31 17:01:18 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.2258322585.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871f28000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31775, start at 2024-05-31 17:01:17.142624033 +0800 CST m=+4.280730365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table mark.finish_mark_3 not exists for 21-th check, retry later table mark.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787367c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32106, start at 2024-05-31 17:01:18.640447968 +0800 CST m=+4.217459258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0036384e0 TaskPositions:map[a6b631e4-0b73-415f-94b9-6a6c45aabed9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b631e4-0b73-415f-94b9-6a6c45aabed9 {"id":"a6b631e4-0b73-415f-94b9-6a6c45aabed9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bb1 a6b631e4-0b73-415f-94b9-6a6c45aabed9 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139541871788041,"min-table-barrier-ts":450139541884895241,"admin-job-type":0} /tidb/cdc/default/default/task/position/a6b631e4-0b73-415f-94b9-6a6c45aabed9/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0036384e0 TaskPositions:map[a6b631e4-0b73-415f-94b9-6a6c45aabed9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b631e4-0b73-415f-94b9-6a6c45aabed9 {"id":"a6b631e4-0b73-415f-94b9-6a6c45aabed9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bb1 a6b631e4-0b73-415f-94b9-6a6c45aabed9 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139541871788041,"min-table-barrier-ts":450139541884895241,"admin-job-type":0} /tidb/cdc/default/default/task/position/a6b631e4-0b73-415f-94b9-6a6c45aabed9/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0036384e0 TaskPositions:map[a6b631e4-0b73-415f-94b9-6a6c45aabed9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d22039c1-63b5-404f-98c2-c01fc42d6ead:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b631e4-0b73-415f-94b9-6a6c45aabed9 {"id":"a6b631e4-0b73-415f-94b9-6a6c45aabed9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/d22039c1-63b5-404f-98c2-c01fc42d6ead {"id":"d22039c1-63b5-404f-98c2-c01fc42d6ead","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59b7f d22039c1-63b5-404f-98c2-c01fc42d6ead /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bb1 a6b631e4-0b73-415f-94b9-6a6c45aabed9 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139541871788041,"min-table-barrier-ts":450139541884895241,"admin-job-type":0} /tidb/cdc/default/default/task/position/a6b631e4-0b73-415f-94b9-6a6c45aabed9/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d22039c1-63b5-404f-98c2-c01fc42d6ead/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "d22039c1-63b5-404f-98c2-c01fc42d6ead" | grep id start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... "cluster-id": "default" "id": "a6b631e4-0b73-415f-94b9-6a6c45aabed9", "cluster-id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/15502b00-b9c0-41e8-b97c-a028c6f427b3 {ClusterID:default ID:default/15502b00-b9c0-41e8-b97c-a028c6f427b3 Info:{"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} Status:0xc00321ac18 TaskPositions:map[839ce72b-1327-47a8-80d5-85dc7f876ba1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ac0e3506-766a-45c7-9aa8-887115d82536:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ac0e3506-766a-45c7-9aa8-887115d82536 {"id":"ac0e3506-766a-45c7-9aa8-887115d82536","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee1a9 ac0e3506-766a-45c7-9aa8-887115d82536 /tidb/cdc/default/default/changefeed/info/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} /tidb/cdc/default/default/changefeed/status/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":450139542133669891,"min-table-barrier-ts":450139542133669891,"admin-job-type":0} /tidb/cdc/default/default/task/position/839ce72b-1327-47a8-80d5-85dc7f876ba1/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ac0e3506-766a-45c7-9aa8-887115d82536/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085753114098292 {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/15502b00-b9c0-41e8-b97c-a028c6f427b3 {ClusterID:default ID:default/15502b00-b9c0-41e8-b97c-a028c6f427b3 Info:{"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} Status:0xc00321ac18 TaskPositions:map[839ce72b-1327-47a8-80d5-85dc7f876ba1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ac0e3506-766a-45c7-9aa8-887115d82536:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/ac0e3506-766a-45c7-9aa8-887115d82536 {"id":"ac0e3506-766a-45c7-9aa8-887115d82536","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee1a9 ac0e3506-766a-45c7-9aa8-887115d82536 /tidb/cdc/default/default/changefeed/info/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use+ grep -q 'failed to get info:' -file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} /tidb/cdc/default/default/changefeed/status/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":450139542133669891,"min-table-barrier-ts":450139542133669891,"admin-job-type":0} /tidb/cdc/default/default/task/position/839ce72b-1327-47a8-80d5-85dc7f876ba1/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ac0e3506-766a-45c7-9aa8-887115d82536/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085753114098292 {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/15502b00-b9c0-41e8-b97c-a028c6f427b3 VARIABLE_NAME VARIABLE_VALUE COMMENT {ClusterID:default ID:default/15502b00-b9c0-41e8-b97c-a028c6f427b3 Info:{"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} Status:0xc00321ac18 TaskPositions:map[839ce72b-1327-47a8-80d5-85dc7f876ba1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ac0e3506-766a-45c7-9aa8-887115d82536:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} bootstrapped True Bootstrap flag. Do not delete. *** etcd info ***: tidb_server_version 110 Bootstrap version. Do not delete. /tidb/cdc/default/__cdc_meta__/capture/839ce72b-1327-47a8-80d5-85dc7f876ba1 system_tz Asia/Shanghai TiDB Global System Timezone. {"id":"839ce72b-1327-47a8-80d5-85dc7f876ba1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} new_collation_enabled True If the new collations are enabled. Do not edit it. /tidb/cdc/default/__cdc_meta__/capture/ac0e3506-766a-45c7-9aa8-887115d82536 tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) {"id":"ac0e3506-766a-45c7-9aa8-887115d82536","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee08e 839ce72b-1327-47a8-80d5-85dc7f876ba1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0aee1a9 ac0e3506-766a-45c7-9aa8-887115d82536 tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) /tidb/cdc/default/default/changefeed/info/15502b00-b9c0-41e8-b97c-a028c6f427b3 tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. {"upstream-id":7375085753114098292,"namespace":"default","changefeed-id":"15502b00-b9c0-41e8-b97c-a028c6f427b3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:00:19.004218152+08:00","start-ts":450139525985337347,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139525998444549} tikv_gc_leader_uuid 63f3785faf8001b Current GC worker leader UUID. (DO NOT EDIT) /tidb/cdc/default/default/changefeed/status/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":450139542133669891,"min-table-barrier-ts":450139542133669891,"admin-job-type":0} /tidb/cdc/default/default/task/position/839ce72b-1327-47a8-80d5-85dc7f876ba1/15502b00-b9c0-41e8-b97c-a028c6f427b3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32513, start at 2024-05-31 17:00:58.479167539 +0800 CST m=+4.310229782 Host name and pid of current GC leader. (DO NOT EDIT) /tidb/cdc/default/default/task/position/ac0e3506-766a-45c7-9aa8-887115d82536/15502b00-b9c0-41e8-b97c-a028c6f427b3 tikv_gc_leader_lease 20240531-17:02:58.485 +0800 Current GC worker leader lease. (DO NOT EDIT) {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085753114098292 tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. {"id":7375085753114098292,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' tikv_gc_last_run_time 20240531-17:00:58.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff successfully nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 22-th check, retry later Starting CDC state checker... table mark.finish_mark exists check diff successfully # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:01:22 CST 2024] <<<<<< run test case multi_source success! >>>>>> Starting CDC state checker... check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 23-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 1-th time, retry later check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 24-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= check diff failed 3-th time, retry later check diff failed 2-th time, retry later wait process 30032 exit for 1-th time... wait process 30032 exit for 2-th time... table mark.finish_mark_3 not exists for 25-th check, retry later nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 30032 exit for 3-th time... wait process 30032 exit for 4-th time... check diff failed 3-th time, retry later The 1 times to try to start tidb cluster... Verifying downstream PD is started... wait process 30032 exit for 5-th time... check diff failed 4-th time, retry later table mark.finish_mark_3 not exists for 26-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (30032) - No such process wait process 30032 exit for 6-th time... process 30032 already exit [Fri May 31 17:01:29 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.3024230244.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 5-th time, retry later table mark.finish_mark_3 not exists for 27-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee380005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37304, start at 2024-05-31 17:01:30.388017309 +0800 CST m=+4.422501948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Attempting encryption of all config, IAM users and policies on MinIO backend + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide table mark.finish_mark_3 not exists for 28-th check, retry later check diff failed 6-th time, retry later Starting Upstream TiDB... * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D487462BA317EB < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 09:01:33 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_capture [Fri May 31 17:01:33 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4338143383.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 29-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff successfully start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc002128b40 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139542987472933,"min-table-barrier-ts":450139542987472933,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc002128b40 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc+ grep -q 'failed to get info:' /default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139542987472933,"min-table-barrier-ts":450139542987472933,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc002128b40 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139542987472933,"min-table-barrier-ts":450139542987472933,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri May 31 17:01:36 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.3031330315.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037ba3c0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 82, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 84, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139546341343233,"min-table-barrier-ts":450139546354450434,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037ba3c0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 82, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 84, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-fi+ grep -q 'failed to get info:' le-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139546341343233,"min-table-barrier-ts":450139546354450434,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0037ba3c0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 82, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating tableID: 84, resolvedTs: 450139546341343233, checkpointTs: 450139546341343233, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139546341343233,"min-table-barrier-ts":450139546354450434,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d3c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22170, start at 2024-05-31 17:00:39.581566015 +0800 CST m=+6.052930553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3784d2500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:22204, start at 2024-05-31 17:00:39.492835094 +0800 CST m=+5.839851040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:39.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:39.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:39.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 1-th time, retry later "is-owner": true, run task successfully ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25761.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release owner pid: 43386 owner id ae3ea623-c958-4b59-bb4f-c1c466f34a6a [Fri May 31 17:01:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4353043532.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info table mark.finish_mark_3 not exists for 30-th check, retry later * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... + set +x + tso='450139546590380033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139546590380033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + start_ts=450139546590380033 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:01:37 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2579025792.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 31-th check, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785fa580017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32411, start at 2024-05-31 17:00:58.408235803 +0800 CST m=+4.330138303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3785faf8001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--nqq2t, pid:32513, start at 2024-05-31 17:00:58.479167539 +0800 CST m=+4.310229782 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:02:58.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:00:58.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:50:58.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:01:40 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3676236764.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc0 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc0047afc68 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139543013425184,"min-table-barrier-ts":450139543013425184,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc0047afc68 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139543013425184,"min-table-barrier-ts":450139543013425184,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} Status:0xc0047afc68 TaskPositions:map[7f1935db-f63d-4b72-8fcd-37c5f13fede5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f1935db-f63d-4b72-8fcd-37c5f13fede5 {"id":"7f1935db-f63d-4b72-8fcd-37c5f13fede5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde0dec1dc 7f1935db-f63d-4b72-8fcd-37c5f13fede5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375085811935213959,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:00:31.166648393+08:00","start-ts":450139529159376902,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139529185329157} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450139543013425184,"min-table-barrier-ts":450139543013425184,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f1935db-f63d-4b72-8fcd-37c5f13fede5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085811935213959 {"id":7375085811935213959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0035b1ad0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 {"id":"da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c29 da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139547127775236,"min-table-barrier-ts":450139547140882435,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0035b1ad0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 {"id":"da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c29 da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139547127775236,"min-table-barrier-ts":450139547140882435,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0035b1ad0 TaskPositions:map[ae3ea623-c958-4b59-bb4f-c1c466f34a6a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae3ea623-c958-4b59-bb4f-c1c466f34a6a {"id":"ae3ea623-c958-4b59-bb4f-c1c466f34a6a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 {"id":"da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59bf8 ae3ea623-c958-4b59-bb4f-c1c466f34a6a /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c29 da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139547127775236,"min-table-barrier-ts":450139547140882435,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae3ea623-c958-4b59-bb4f-c1c466f34a6a/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "ae3ea623-c958-4b59-bb4f-c1c466f34a6a" | grep id "cluster-id": "default" "id": "da1d2bf3-f848-4ac0-bf0d-5b2fb8e120a3", "cluster-id": "default" run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871f28000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31775, start at 2024-05-31 17:01:17.142624033 +0800 CST m=+4.280730365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... Starting Upstream TiDB... nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 32-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de88f4e4-c670-411d-8199-f36262f32993 {"id":"de88f4e4-c670-411d-8199-f36262f32993","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1e62992 de88f4e4-c670-411d-8199-f36262f32993 /tidb/cdc/default/default/upstream/7375086043065565949 {"id":7375086043065565949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de88f4e4-c670-411d-8199-f36262f32993 {"id":"de88f4e4-c670-411d-8199-f36262f32993","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1e62992 de88f4e4-c670-411d-8199-f36262f32993 /tidb/cdc/default/default/upstream/7375086043065565949 {"id":7375086043065565949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/de88f4e4-c670-411d-8199-f36262f32993 {"id":"de88f4e4-c670-411d-8199-f36262f32993","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1e62992 de88f4e4-c670-411d-8199-f36262f32993 /tidb/cdc/default/default/upstream/7375086043065565949 {"id":7375086043065565949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139546590380033 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25834.out cli changefeed create --start-ts=450139546590380033 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 3-th time, retry later Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375086043065565949,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:01:41.613330059+08:00","start_ts":450139546590380033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139546590380033,"checkpoint_ts":450139546590380033,"checkpoint_time":"2024-05-31 17:01:37.528"} PASS cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:01:41 CST 2024] <<<<<< run test case owner_resign success! >>>>>> coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 2-th check, retry later + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1920 0 --:--:-- --:--:-- --:--:-- 1938 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:01:37.528","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 17:01:42.000","info":"Data syncing is finished"}' ++ jq .synced ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:01:37.528","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:01:42.000","info":"Data' syncing is 'finished"}' + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:01:37.528","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:01:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-31 17:01:37.528' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:01:37.528","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:01:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888db80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24492, start at 2024-05-31 17:01:40.619122568 +0800 CST m=+4.263413587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:01:37.528","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:01:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-31 17:01:42' + echo 'sink_checkpoint_ts is 2024-05-31' 17:01:37.528 sink_checkpoint_ts is 2024-05-31 17:01:37.528 ++ date -d '2024-05-31 17:01:37.528' +%s + checkpoint_timestamp=1717146097 ++ date -d '2024-05-31 17:01:42' +%s + current_timestamp=1717146102 + '[' 5 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 33-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:01:43 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3681136813.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787367c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32106, start at 2024-05-31 17:01:18.640447968 +0800 CST m=+4.217459258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 34-th check, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:46 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri May 31 17:01:46 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3687036872.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 35-th check, retry later check diff successfully nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later table ddl_manager.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f634000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19995, start at 2024-05-31 17:01:47.288296868 +0800 CST m=+4.305462326 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.277 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.277 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log table mark.finish_mark_3 not exists for 36-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:49 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/187dde54-8143-4229-83c2-3a412d1e655a {"id":"187dde54-8143-4229-83c2-3a412d1e655a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11c2 187dde54-8143-4229-83c2-3a412d1e655a /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/187dde54-8143-4229-83c2-3a412d1e655a {"id":"187dde54-8143-4229-83c2-3a412d1e655a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11c2 187dde54-8143-4229-83c2-3a412d1e655a /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/187dde54-8143-4229-83c2-3a412d1e655a {"id":"187dde54-8143-4229-83c2-3a412d1e655a","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/a584fb55-7f6d-4e68-ba18-705135f173a0 {"id":"a584fb55-7f6d-4e68-ba18-705135f173a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e884295b-fb42-40cc-bba6-bea8a00566c9 {"id":"e884295b-fb42-40cc-bba6-bea8a00566c9","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb118d a584fb55-7f6d-4e68-ba18-705135f173a0 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11b0 e884295b-fb42-40cc-bba6-bea8a00566c9 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1fb11c2 187dde54-8143-4229-83c2-3a412d1e655a /tidb/cdc/default/default/upstream/7375086128377288249 {"id":7375086128377288249,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.cli.36939.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 8e5f40b6-40c7-4431-b37c-fffc5c24b9ab Info: {"upstream_id":7375086128377288249,"namespace":"default","id":"8e5f40b6-40c7-4431-b37c-fffc5c24b9ab","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-31T17:01:49.642404573+08:00","start_ts":450139549747380226,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139549747380226,"checkpoint_ts":450139549747380226,"checkpoint_time":"2024-05-31 17:01:49.571"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table ddl_manager.finish_mark not exists for 6-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ***************** properties ***************** "readallfields"="true" "threadcount"="10" "mysql.host"="127.0.0.1" "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "workload"="core" "mysql.port"="4000" "requestdistribution"="uniform" "recordcount"="10000" "scanproportion"="0" "mysql.db"="move_table" "operationcount"="0" "dotransactions"="false" "insertproportion"="0" ********************************************** start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... INSERT - Takes(s): 10.0, Count: 8554, OPS: 855.8, Avg(us): 11648, Min(us): 795, Max(us): 405197, 95th(us): 87000, 99th(us): 191000 Starting CDC state checker... table mark.finish_mark_3 not exists for 37-th check, retry later check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Run finished, takes 10.417407082s INSERT - Takes(s): 10.4, Count: 10000, OPS: 960.4, Avg(us): 10366, Min(us): 763, Max(us): 405197, 95th(us): 86000, 99th(us): 184000 [Fri May 31 17:01:51 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2286022862.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_capture [Fri May 31 17:01:53 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4374343745.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 38-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 8-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee380005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37304, start at 2024-05-31 17:01:30.388017309 +0800 CST m=+4.422501948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.22925.out cli changefeed create --start-ts=450139532321095681 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info: {"upstream_id":7375085875200933946,"namespace":"default","id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:01:55.188943282+08:00","start_ts":450139532321095681,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139532321095681,"checkpoint_ts":450139532321095681,"checkpoint_time":"2024-05-31 17:00:43.095"} PASS start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:01:55 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2296122963.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 39-th check, retry later table ddl_manager.finish_mark not exists for 9-th check, retry later check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0019e06a8 TaskPositions:map[ee10344d-b9d7-4f73-873c-1d4e01c251e4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 82, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 84, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee10344d-b9d7-4f73-873c-1d4e01c251e4 {"id":"ee10344d-b9d7-4f73-873c-1d4e01c251e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c6e ee10344d-b9d7-4f73-873c-1d4e01c251e4 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139551583961093,"min-table-barrier-ts":450139551597068291,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee10344d-b9d7-4f73-873c-1d4e01c251e4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0019e06a8 TaskPositions:map[ee10344d-b9d7-4f73-873c-1d4e01c251e4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 82, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 84, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee10344d-b9d7-4f73-873c-1d4e01c251e4 {"id":"ee10344d-b9d7-4f73-873c-1d4e01c251e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c6e ee10344d-b9d7-4f73-873c-1d4e01c251e4 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139551583961093,"min-table-barrier-ts":450139551597068291,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee10344d-b9d7-4f73-873c-1d4e01c251e4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0019e06a8 TaskPositions:map[ee10344d-b9d7-4f73-873c-1d4e01c251e4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 82, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating tableID: 84, resolvedTs: 450139551583961093, checkpointTs: 450139551583961093, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee10344d-b9d7-4f73-873c-1d4e01c251e4 {"id":"ee10344d-b9d7-4f73-873c-1d4e01c251e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c6e ee10344d-b9d7-4f73-873c-1d4e01c251e4 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} + grep -q 'etcd info' /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139551583961093,"min-table-barrier-ts":450139551597068291,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee10344d-b9d7-4f73-873c-1d4e01c251e4/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 43748 owner id ee10344d-b9d7-4f73-873c-1d4e01c251e4 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'ee10344d-b9d7-4f73-873c-1d4e01c251e4' /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59c6e run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 40-th check, retry later check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:01:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc0011cb050 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547564507163,"min-table-barrier-ts":450139551942311940,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc0011cb050 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-wor+ grep -q 'failed to get info:' ker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547564507163,"min-table-barrier-ts":450139551942311940,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc0011cb050 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547564507163,"min-table-barrier-ts":450139551942311940,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri May 31 17:01:58 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2302123023.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 41-th check, retry later check diff failed 6-th time, retry later nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully table ddl_manager.finish_mark not exists for 11-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Fri May 31 17:02:00 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4395643958.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871e70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31674, start at 2024-05-31 17:01:17.099902368 +0800 CST m=+4.325774232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37871f28000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:31775, start at 2024-05-31 17:01:17.142624033 +0800 CST m=+4.280730365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:17.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:17.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 42-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc002984b58 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cfb524df-a2e1-410a-af26-b2a448b86c59:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cfb524df-a2e1-410a-af26-b2a448b86c59 {"id":"cfb524df-a2e1-410a-af26-b2a448b86c59","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180575 cfb524df-a2e1-410a-af26-b2a448b86c59 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547616936082,"min-table-barrier-ts":450139552729006088,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cfb524df-a2e1-410a-af26-b2a448b86c59/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc002984b58 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cfb524df-a2e1-410a-af26-b2a448b86c59:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cfb524df-a2e1-410a-af26-b2a448b86c59 {"id":"cfb524df-a2e1-410a-af26-b2a448b86c59","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180575 cfb524df-a2e1-410a-af26-b2a448b86c59 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547616936082,"min-table-barrier-ts":450139552729006088,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cfb524df-a2e1-410a-af26-b2a448b86c59/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {ClusterID:default ID:default/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa Info:{"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} Status:0xc002984b58 TaskPositions:map[859e534a-2b7a-4135-9597-ed3585984108:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cb6f5484-9dfa-4b19-b163-e1d644a2d6d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} cfb524df-a2e1-410a-af26-b2a448b86c59:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/859e534a-2b7a-4135-9597-ed3585984108 {"id":"859e534a-2b7a-4135-9597-ed3585984108","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 {"id":"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/cfb524df-a2e1-410a-af26-b2a448b86c59 {"id":"cfb524df-a2e1-410a-af26-b2a448b86c59","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180525 859e534a-2b7a-4135-9597-ed3585984108 /tidb/cdc/default/__cdc_meta__/owner/22318fcde118055b cb6f5484-9dfa-4b19-b163-e1d644a2d6d1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde1180575 cfb524df-a2e1-410a-af26-b2a448b86c59 /tidb/cdc/default/default/changefeed/info/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"upstream-id":7375085875200933946,"namespace":"default","changefeed-id":"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:01:55.188943282+08:00","start-ts":450139532321095681,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139551208570886} /tidb/cdc/default/default/changefeed/status/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":450139547616936082,"min-table-barrier-ts":450139552729006088,"admin-job-type":0} /tidb/cdc/default/default/task/position/859e534a-2b7a-4135-9597-ed3585984108/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb6f5484-9dfa-4b19-b163-e1d644a2d6d1/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cfb524df-a2e1-410a-af26-b2a448b86c59/13619ce4-8c4c-4d62-ac58-7dc9b1de62aa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085875200933946 {"id":7375085875200933946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 12-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... table move_table.usertable exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378736a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32015, start at 2024-05-31 17:01:18.656175527 +0800 CST m=+4.328244932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787367c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:32106, start at 2024-05-31 17:01:18.640447968 +0800 CST m=+4.217459258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:18.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:18.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:18.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:02:02 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> [Fri May 31 17:02:03 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS= + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3612536127.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013e30e0 TaskPositions:map[e16dfc8b-b5e3-417b-b40b-5a8f520c8e01:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 82, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 84, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 {"id":"e16dfc8b-b5e3-417b-b40b-5a8f520c8e01","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ca4 e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139553157087238,"min-table-barrier-ts":450139553157087238,"admin-job-type":0} /tidb/cdc/default/default/task/position/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013e30e0 TaskPositions:map[e16dfc8b-b5e3-417b-b40b-5a8f520c8e01:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 82, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 84, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 {"id":"e16dfc8b-b5e3-417b-b40b-5a8f520c8e01","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ca4 e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139553157087238,"min-table-barrier-ts":450139553157087238,"admin-job-type":0} /tidb/cdc/default/default/task/position/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {ClusterID:default ID:default/c17a8d8c-aa23-4a58-96bb-84543c50aa7a Info:{"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} Status:0xc0013e30e0 TaskPositions:map[e16dfc8b-b5e3-417b-b40b-5a8f520c8e01:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 82, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating tableID: 84, resolvedTs: 450139553157087238, checkpointTs: 450139553157087238, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 {"id":"e16dfc8b-b5e3-417b-b40b-5a8f520c8e01","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcddec59ca4 e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 /tidb/cdc/default/default/changefeed/info/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"upstream-id":7375085185851773928,"namespace":"default","changefeed-id":"c17a8d8c-aa23-4a58-96bb-84543c50aa7a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T16:59:10.914970049+08:00","start-ts":450139492064428033,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139508146962438} /tidb/cdc/default/default/changefeed/status/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":450139553157087238,"min-table-barrier-ts":450139553157087238,"admin-job-type":0} /tidb/cdc/default/default/task/position/e16dfc8b-b5e3-417b-b40b-5a8f520c8e01/c17a8d8c-aa23-4a58-96bb-84543c50aa7a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375085185851773928 {"id":7375085185851773928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully table mark.finish_mark_3 not exists for 43-th check, retry later split_and_random_merge scale: 20 table ddl_manager.finish_mark not exists for 13-th check, retry later owner pid: 43961 owner id e16dfc8b-b5e3-417b-b40b-5a8f520c8e01 start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... changefeed id: c17a8d8c-aa23-4a58-96bb-84543c50aa7a curl -s -d "cf-id=c17a8d8c-aa23-4a58-96bb-84543c50aa7a&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 581 897 --:--:-- --:--:-- --:--:-- 900 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888db80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24492, start at 2024-05-31 17:01:40.619122568 +0800 CST m=+4.263413587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 44-th check, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15c5d260-2e6a-4089-804c-2580a5b70f26 {"id":"15c5d260-2e6a-4089-804c-2580a5b70f26","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde253398e 15c5d260-2e6a-4089-804c-2580a5b70f26 /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15c5d260-2e6a-4089-804c-2580a5b70f26 {"id":"15c5d260-2e6a-4089-804c-2580a5b70f26","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde253398e 15c5d260-2e6a-4089-804c-2580a5b70f26 /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15c5d260-2e6a-4089-804c-2580a5b70f26 {"id":"15c5d260-2e6a-4089-804c-2580a5b70f26","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde253398e 15c5d260-2e6a-4089-804c-2580a5b70f26 /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later Starting TLS TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... table mark.finish_mark_3 not exists for 45-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later [2024/05/31 17:02:07.819 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/31 17:02:07.821 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/31 17:02:07.822 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=859e534a-2b7a-4135-9597-ed3585984108] [2024/05/31 17:02:07.822 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/31 17:02:07.823 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/31 17:02:07.966 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=859e534a-2b7a-4135-9597-ed3585984108] [processorDetail="{\"table_ids\":[82]}"] [2024/05/31 17:02:08.122 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=cb6f5484-9dfa-4b19-b163-e1d644a2d6d1] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:02:08.288 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=cfb524df-a2e1-410a-af26-b2a448b86c59] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:02:08.288 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"859e534a-2b7a-4135-9597-ed3585984108\":[{\"ID\":82,\"Changefeed\":\"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa\"}],\"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1\":[],\"cfb524df-a2e1-410a-af26-b2a448b86c59\":[]}"] [2024/05/31 17:02:08.288 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa&target-cp-id=cfb524df-a2e1-410a-af26-b2a448b86c59&table-id=82"] [2024/05/31 17:02:08.330 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=82] [2024/05/31 17:02:08.330 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=859e534a-2b7a-4135-9597-ed3585984108] [targetCapture=cfb524df-a2e1-410a-af26-b2a448b86c59] table move_table.check1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) *************************** 1. row *************************** count(distinct region_id): 6 Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 46-th check, retry later Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 16-th check, retry later table move_table.check1 not exists for 2-th check, retry later nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:02:11 CST 2024] <<<<<< run test case availability success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f634000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19995, start at 2024-05-31 17:01:47.288296868 +0800 CST m=+4.305462326 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.277 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.277 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 47-th check, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check1 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) split_and_random_merge scale: 40 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a714c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:36293, start at 2024-05-31 17:02:11.543454968 +0800 CST m=+4.298427610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:11.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:11.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:11.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... table mark.finish_mark_3 not exists for 48-th check, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 2 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37209, start at 2024-05-31 17:01:30.400836878 +0800 CST m=+4.482728353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3787ee380005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:37304, start at 2024-05-31 17:01:30.388017309 +0800 CST m=+4.422501948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:30.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:30.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:30.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:02:14 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.4147641478.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table move_table.check1 not exists for 4-th check, retry later Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7a300008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23843, start at 2024-05-31 17:02:12.119105457 +0800 CST m=+13.145352523 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.108 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.108 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log table mark.finish_mark_3 not exists for 49-th check, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later table move_table.check1 exists Starting TLS TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a714c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:36293, start at 2024-05-31 17:02:11.543454968 +0800 CST m=+4.298427610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:11.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:11.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:11.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 17:02:16.699 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/05/31 17:02:16.699 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] Starting CDC state checker... check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:17 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/395c7b3a-f91a-40e3-b66b-2ffe6dba4570 {"id":"395c7b3a-f91a-40e3-b66b-2ffe6dba4570","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde27d2c91 395c7b3a-f91a-40e3-b66b-2ffe6dba4570 /tidb/cdc/default/default/upstream/7375086265666419635 {"id":7375086265666419635,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/395c7b3a-f91a-40e3-b66b-2ffe6dba4570 {"id":"395c7b3a-f91a-40e3-b66b-2ffe6dba4570","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde27d2c91 395c7b3a-f91a-40e3-b66b-2ffe6dba4570 /tidb/cdc/default/default/upstream/7375086265666419635 {"id":7375086265666419635,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/395c7b3a-f91a-40e3-b66b-2ffe6dba4570 {"id":"395c7b3a-f91a-40e3-b66b-2ffe6dba4570","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde27d2c91 395c7b3a-f91a-40e3-b66b-2ffe6dba4570 /tidb/cdc/default/default/upstream/7375086265666419635 {"id":7375086265666419635,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.41550.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: 81fbda08-e66e-478d-abad-c7f66ef9cc79 Info: {"upstream_id":7375086265666419635,"namespace":"default","id":"81fbda08-e66e-478d-abad-c7f66ef9cc79","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-05-31T17:02:17.756654291+08:00","start_ts":450139557115461633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139557115461633,"checkpoint_ts":450139557115461633,"checkpoint_time":"2024-05-31 17:02:17.678"} PASS table mark.finish_mark_3 not exists for 50-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 3 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table ddl_manager.finish_mark not exists for 20-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 51-th check, retry later ***************** properties ***************** "workload"="core" "readallfields"="true" "insertproportion"="0" "mysql.port"="4000" "scanproportion"="0" "mysql.db"="sink_retry" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "operationcount"="0" "threadcount"="2" "readproportion"="0" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "recordcount"="10" ********************************************** Run finished, takes 9.232035ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2002.7, Avg(us): 1786, Min(us): 1023, Max(us): 4200, 95th(us): 5000, 99th(us): 5000 [Fri May 31 17:02:09 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.4189641898.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:12 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3549c6c-c38e-4fed-b73f-57d84c1cddec {"id":"d3549c6c-c38e-4fed-b73f-57d84c1cddec","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde187d818 d3549c6c-c38e-4fed-b73f-57d84c1cddec /tidb/cdc/default/default/upstream/7375085995055464013 {"id":7375085995055464013,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3549c6c-c38e-4fed-b73f-57d84c1cddec {"id":"d3549c6c-c38e-4fed-b73f-57d84c1cddec","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde187d818 d3549c6c-c38e-4fed-b73f-57d84c1cddec /tidb/cdc/default/default/upstream/7375085995055464013 {"id":7375085995055464013,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3549c6c-c38e-4fed-b73f-57d84c1cddec {"id":"d3549c6c-c38e-4fed-b73f-57d84c1cddec","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde187d818 d3549c6c-c38e-4fed-b73f-57d84c1cddec /tidb/cdc/default/default/upstream/7375085995055464013 {"id":7375085995055464013,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41959.out cli changefeed create --start-ts=450139539824967681 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 9694f6e6-c98f-48a1-a4e6-d08bdeb931bd Info: {"upstream_id":7375085995055464013,"namespace":"default","id":"9694f6e6-c98f-48a1-a4e6-d08bdeb931bd","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:02:13.063807284+08:00","start_ts":450139539824967681,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139539824967681,"checkpoint_ts":450139539824967681,"checkpoint_time":"2024-05-31 17:01:11.720"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x *************************** 1. row *************************** count(distinct region_id): 19 Starting TLS TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a714c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:36293, start at 2024-05-31 17:02:11.543454968 +0800 CST m=+4.298427610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:11.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:11.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:11.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 17:02:20.171 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/05/31 17:02:20.171 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] Starting CDC state checker... table ddl_manager.finish_mark not exists for 21-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed [Fri May 31 17:02:21 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3822238224.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:02:21.351 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/31 17:02:21.353 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/31 17:02:21.353 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=859e534a-2b7a-4135-9597-ed3585984108] [2024/05/31 17:02:21.354 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/31 17:02:21.354 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/31 17:02:21.466 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=859e534a-2b7a-4135-9597-ed3585984108] [processorDetail="{\"table_ids\":[84,86]}"] [2024/05/31 17:02:21.616 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=cb6f5484-9dfa-4b19-b163-e1d644a2d6d1] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:02:21.766 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa] [captureID=cfb524df-a2e1-410a-af26-b2a448b86c59] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:02:21.766 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"859e534a-2b7a-4135-9597-ed3585984108\":[{\"ID\":84,\"Changefeed\":\"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa\"},{\"ID\":86,\"Changefeed\":\"13619ce4-8c4c-4d62-ac58-7dc9b1de62aa\"}],\"cb6f5484-9dfa-4b19-b163-e1d644a2d6d1\":[],\"cfb524df-a2e1-410a-af26-b2a448b86c59\":[]}"] [2024/05/31 17:02:21.766 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa&target-cp-id=cfb524df-a2e1-410a-af26-b2a448b86c59&table-id=84"] [2024/05/31 17:02:21.816 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=84] [2024/05/31 17:02:21.816 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=13619ce4-8c4c-4d62-ac58-7dc9b1de62aa&target-cp-id=cfb524df-a2e1-410a-af26-b2a448b86c59&table-id=86"] [2024/05/31 17:02:21.866 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=86] [2024/05/31 17:02:21.866 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=859e534a-2b7a-4135-9597-ed3585984108] [targetCapture=cfb524df-a2e1-410a-af26-b2a448b86c59] check diff successfully table move_table.check2 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 52-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 53-th check, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later table move_table.check2 exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:24 GMT < Content-Length: 1049 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:02:24 CST 2024] <<<<<< run test case move_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888c480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24415, start at 2024-05-31 17:01:40.526734613 +0800 CST m=+4.247737161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37888db80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:24492, start at 2024-05-31 17:01:40.619122568 +0800 CST m=+4.263413587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:40.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:40.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:40.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.28410.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139558952828931 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139558952828931 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:02:24 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.2844728449.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) split_and_random_merge scale: 80 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 54-th check, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later [Fri May 31 17:02:26 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3830238304.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4e380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41968, start at 2024-05-31 17:02:25.698817853 +0800 CST m=+4.441033315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 55-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 25-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3577c89d-36de-467f-8e13-b8c004d0da9e {"id":"3577c89d-36de-467f-8e13-b8c004d0da9e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2a7058d 3577c89d-36de-467f-8e13-b8c004d0da9e /tidb/cdc/default/default/upstream/7375086312334072213 {"id":7375086312334072213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3577c89d-36de-467f-8e13-b8c004d0da9e {"id":"3577c89d-36de-467f-8e13-b8c004d0da9e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2a7058d 3577c89d-36de-467f-8e13-b8c004d0da9e /tidb/cdc/default/default/upstream/7375086312334072213 {"id":7375086312334072213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3577c89d-36de-467f-8e13-b8c004d0da9e {"id":"3577c89d-36de-467f-8e13-b8c004d0da9e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2a7058d 3577c89d-36de-467f-8e13-b8c004d0da9e /tidb/cdc/default/default/upstream/7375086312334072213 {"id":7375086312334072213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.28509.out cli changefeed create --start-ts=450139558952828931 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 3860bb6b-40e5-4bdd-884f-e7e4410260b0 Info: {"upstream_id":7375086312334072213,"namespace":"default","id":"3860bb6b-40e5-4bdd-884f-e7e4410260b0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:02:28.419129223+08:00","start_ts":450139558952828931,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139558952828931,"checkpoint_ts":450139558952828931,"checkpoint_time":"2024-05-31 17:02:24.687"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 56-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:29 GMT < Content-Length: 1303 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc286e0-e8fb-4ace-bdfd-ece9de3b346f {"id":"0dc286e0-e8fb-4ace-bdfd-ece9de3b346f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3d3 0dc286e0-e8fb-4ace-bdfd-ece9de3b346f /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc286e0-e8fb-4ace-bdfd-ece9de3b346f {"id":"0dc286e0-e8fb-4ace-bdfd-ece9de3b346f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3d3 0dc286e0-e8fb-4ace-bdfd-ece9de3b346f /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dc286e0-e8fb-4ace-bdfd-ece9de3b346f {"id":"0dc286e0-e8fb-4ace-bdfd-ece9de3b346f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f50d63d7-eea3-48e1-a7a2-8dc727711763 {"id":"f50d63d7-eea3-48e1-a7a2-8dc727711763","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3ae f50d63d7-eea3-48e1-a7a2-8dc727711763 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fcde274c3d3 0dc286e0-e8fb-4ace-bdfd-ece9de3b346f /tidb/cdc/default/default/upstream/7375086444087581372 {"id":7375086444087581372,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 26-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f4c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19908, start at 2024-05-31 17:01:47.214870505 +0800 CST m=+4.308678098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3788f634000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:19995, start at 2024-05-31 17:01:47.288296868 +0800 CST m=+4.305462326 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:03:47.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:01:47.277 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:51:47.277 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:02:31 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.2383023832.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: check health ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get status table mark.finish_mark_3 not exists for 57-th check, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450139560919433218, "checkpoint_time": "2024-05-31 17:02:32.189", "error": null }' + echo '{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450139560919433218, "checkpoint_time": "2024-05-31 17:02:32.189", "error": null }' { "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 450139560919433218, "checkpoint_time": "2024-05-31 17:02:32.189", "error": null } ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 450139560919433218, '"checkpoint_time":' '"2024-05-31' '17:02:32.189",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 450139560919433218, '"checkpoint_time":' '"2024-05-31' '17:02:32.189",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s + info='{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450139560959016962, "checkpoint_time": "2024-05-31 17:02:32.340", "error": null }' + echo '{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450139560959016962, "checkpoint_time": "2024-05-31 17:02:32.340", "error": null }' { "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 450139560959016962, "checkpoint_time": "2024-05-31 17:02:32.340", "error": null } ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 450139560959016962, '"checkpoint_time":' '"2024-05-31' '17:02:32.340",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 450139560959016962, '"checkpoint_time":' '"2024-05-31' '17:02:32.340",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + info='{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450139560985231363, "checkpoint_time": "2024-05-31 17:02:32.440", "error": null }' + echo '{ "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450139560985231363, "checkpoint_time": "2024-05-31 17:02:32.440", "error": null }' { "upstream_id": 7375086444087581372, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 450139560985231363, "checkpoint_time": "2024-05-31 17:02:32.440", "error": null } ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 450139560985231363, '"checkpoint_time":' '"2024-05-31' '17:02:32.440",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086444087581372, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 450139560985231363, '"checkpoint_time":' '"2024-05-31' '17:02:32.440",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb0800016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45105, start at 2024-05-31 17:02:32.001809479 +0800 CST m=+4.436942907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:31.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:31.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log table mark.finish_mark_3 not exists for 58-th check, retry later table ddl_manager.finish_mark not exists for 28-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd2a1ac-9c3f-477c-a930-d782a3c9db5b {"id":"2dd2a1ac-9c3f-477c-a930-d782a3c9db5b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2c13d8e 2dd2a1ac-9c3f-477c-a930-d782a3c9db5b /tidb/cdc/default/default/upstream/7375086329148012408 {"id":7375086329148012408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd2a1ac-9c3f-477c-a930-d782a3c9db5b {"id":"2dd2a1ac-9c3f-477c-a930-d782a3c9db5b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2c13d8e 2dd2a1ac-9c3f-477c-a930-d782a3c9db5b /tidb/cdc/default/default/upstream/7375086329148012408 {"id":7375086329148012408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dd2a1ac-9c3f-477c-a930-d782a3c9db5b {"id":"2dd2a1ac-9c3f-477c-a930-d782a3c9db5b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2c13d8e 2dd2a1ac-9c3f-477c-a930-d782a3c9db5b /tidb/cdc/default/default/upstream/7375086329148012408 {"id":7375086329148012408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7375086329148012408,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:02:34.539212497+08:00","start_ts":450139560687173633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139560687173633,"checkpoint_ts":450139560687173633,"checkpoint_time":"2024-05-31 17:02:31.303"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7375086329148012408,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:02:34.684224102+08:00","start_ts":450139560687173633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139560687173633,"checkpoint_ts":450139560687173633,"checkpoint_time":"2024-05-31 17:02:31.303"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Fri May 31 17:02:33 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7375086329148012408,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:02:34.835708055+08:00","start_ts":450139560687173633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139560687173633,"checkpoint_ts":450139560687173633,"checkpoint_time":"2024-05-31 17:02:31.303"} Starting CDC state checker... table mark.finish_mark_3 not exists for 59-th check, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7a300008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23843, start at 2024-05-31 17:02:12.119105457 +0800 CST m=+13.145352523 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.108 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.108 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 60-th check, retry later table ddl_manager.finish_mark not exists for 30-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3900 0 --:--:-- --:--:-- --:--:-- 3857 100 243 100 243 0 0 3889 0 --:--:-- --:--:-- --:--:-- 3857 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:02:38.828","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:02:32.429","now_ts":"2024-05-31 17:02:39.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:02:38.828","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:02:32.429","now_ts":"2024-05-31' '17:02:39.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:02:38.828","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:02:32.429","now_ts":"2024-05-31' '17:02:39.000","info":"The' data syncing is not finished, please 'wait"}' + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table mark.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 32-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:02:43 CST 2024] <<<<<< run test case default_value success! >>>>>> table ddl_manager.finish_mark not exists for 33-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + workdir=/tmp/tidb_cdc_test/canal_json_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_partition_table ++ date + echo '[Fri May 31 17:02:44 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>>' [Fri May 31 17:02:44 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "mysql.user"="root" "insertproportion"="0" "dotransactions"="false" "recordcount"="10" "threadcount"="2" "mysql.db"="sink_retry" "readallfields"="true" "workload"="core" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readproportion"="0" "operationcount"="0" "scanproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 4.79044ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3393.3, Avg(us): 897, Min(us): 531, Max(us): 1824, 95th(us): 2000, 99th(us): 2000 table ddl_manager.finish_mark not exists for 34-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4e380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41968, start at 2024-05-31 17:02:25.698817853 +0800 CST m=+4.441033315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 36-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd5240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25243, start at 2024-05-31 17:02:50.704768663 +0800 CST m=+4.550382704 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:02:48 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table partition_table.t not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... table partition_table.t not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 38-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb0800016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45105, start at 2024-05-31 17:02:32.001809479 +0800 CST m=+4.436942907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:31.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:31.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7bb4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23769, start at 2024-05-31 17:02:12.220388823 +0800 CST m=+13.320002756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378a7a300008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:23843, start at 2024-05-31 17:02:12.119105457 +0800 CST m=+13.145352523 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:12.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:12.108 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:12.108 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.27716.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 39-th check, retry later + set +x + tso='450139567287173121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139567287173121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:02:56 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.2774927751.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Fri May 31 17:02:57 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> table ddl_manager.finish_mark not exists for 40-th check, retry later table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:02:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e12b7c89-7381-4d05-9655-7e27e507ecbe {"id":"e12b7c89-7381-4d05-9655-7e27e507ecbe","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde322ca90 e12b7c89-7381-4d05-9655-7e27e507ecbe /tidb/cdc/default/default/upstream/7375086408282582156 {"id":7375086408282582156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e12b7c89-7381-4d05-9655-7e27e507ecbe {"id":"e12b7c89-7381-4d05-9655-7e27e507ecbe","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde322ca90 e12b7c89-7381-4d05-9655-7e27e507ecbe /tidb/cdc/default/default/upstream/7375086408282582156 {"id":7375086408282582156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e12b7c89-7381-4d05-9655-7e27e507ecbe {"id":"e12b7c89-7381-4d05-9655-7e27e507ecbe","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde322ca90 e12b7c89-7381-4d05-9655-7e27e507ecbe /tidb/cdc/default/default/upstream/7375086408282582156 {"id":7375086408282582156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.27814.out cli changefeed create --start-ts=450139567287173121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 46d60604-21ee-45a4-896d-2fabc7ec2050 Info: {"upstream_id":7375086408282582156,"namespace":"default","id":"46d60604-21ee-45a4-896d-2fabc7ec2050","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:03:00.194533242+08:00","start_ts":450139567287173121,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139567287173121,"checkpoint_ts":450139567287173121,"checkpoint_time":"2024-05-31 17:02:56.480"} PASS table ddl_manager.finish_mark not exists for 41-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table partition_table.finish_mark not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 42-th check, retry later table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed pass test: resume changefeed pass test: rebalance table pass test: list processors pass test: get processors pass test: move table pass test: set log level ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... pass test: remove changefeed Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: resign owner pass test: get tso pass test: verify table table ddl_manager.finish_mark not exists for 44-th check, retry later pass test: create changefeed v2 status code 200 pass test: show metadata status code 204 status code 204 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock table partition_table.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc380000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:19062, start at 2024-05-31 17:03:05.966450551 +0800 CST m=+4.676971598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:03:07 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4be80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41870, start at 2024-05-31 17:02:25.556175565 +0800 CST m=+4.388740019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378b4e380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:41968, start at 2024-05-31 17:02:25.698817853 +0800 CST m=+4.441033315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:25.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:25.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:25.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:03:08 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.4625146253.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 45-th check, retry later table partition_table.finish_mark not exists for 7-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 46-th check, retry later table partition_table.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:03:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7406c3b7-2945-4ffe-bb92-791a3c60d09a {"id":"7406c3b7-2945-4ffe-bb92-791a3c60d09a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553e8e 7406c3b7-2945-4ffe-bb92-791a3c60d09a /tidb/cdc/default/default/upstream/7375086507879429105 {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7406c3b7-2945-4ffe-bb92-791a3c60d09a {"id":"7406c3b7-2945-4ffe-bb92-791a3c60d09a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553e8e 7406c3b7-2945-4ffe-bb92-791a3c60d09a /tidb/cdc/default/default/upstream/7375086507879429105 {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7406c3b7-2945-4ffe-bb92-791a3c60d09a {"id":"7406c3b7-2945-4ffe-bb92-791a3c60d09a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553e8e 7406c3b7-2945-4ffe-bb92-791a3c60d09a /tidb/cdc/default/default/upstream/7375086507879429105 {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 47-th check, retry later table partition_table.finish_mark not exists for 9-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table2.t2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later table partition_table2.t2 not exists for 3-th check, retry later table partition_table2.t2 not exists for 4-th check, retry later table partition_table2.t2 not exists for 5-th check, retry later + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Fri May 31 17:03:11 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Fri May 31 17:03:11 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml table partition_table2.t2 not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 48-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb23c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45008, start at 2024-05-31 17:02:32.127219444 +0800 CST m=+4.611422832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:32.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:32.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378bb0800016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--vf9n7, pid:45105, start at 2024-05-31 17:02:32.001809479 +0800 CST m=+4.436942907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:32.008 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:31.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:31.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd5240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25243, start at 2024-05-31 17:02:50.704768663 +0800 CST m=+4.550382704 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed table ddl_manager.finish_mark not exists for 49-th check, retry later table partition_table.finish_mark not exists for 11-th check, retry later table partition_table2.t2 not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.72.250 + export UP_TIDB_HOST=10.233.72.250 UP_PD_HOST_1=10.233.72.250 UP_PD_HOST_2=10.233.72.250 UP_PD_HOST_3=10.233.72.250 UP_TIKV_HOST_1=10.233.72.250 UP_TIKV_HOST_2=10.233.72.250 UP_TIKV_HOST_3=10.233.72.250 + UP_TIDB_HOST=10.233.72.250 + UP_PD_HOST_1=10.233.72.250 + UP_PD_HOST_2=10.233.72.250 + UP_PD_HOST_3=10.233.72.250 + UP_TIKV_HOST_1=10.233.72.250 + UP_TIKV_HOST_2=10.233.72.250 + UP_TIKV_HOST_3=10.233.72.250 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=10509 + trap 'stop_tidb_cluster && stop_proxy' EXIT + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.72.250' UP_TIDB_HOST 10.233.72.250 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.72.250' UP_PD_HOST_1 10.233.72.250 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.72.250' UP_PD_HOST_2 10.233.72.250 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.72.250' UP_PD_HOST_3 10.233.72.250 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.72.250' UP_TIKV_HOST_1 10.233.72.250 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.72.250' UP_TIKV_HOST_2 10.233.72.250 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.72.250' UP_TIKV_HOST_3 10.233.72.250 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster table sink_retry.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:03:14 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table partition_table2.t2 not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later table partition_table.finish_mark not exists for 12-th check, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table2.t2 exists table ddl_manager.finish_mark not exists for 51-th check, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 13-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 52-th check, retry later table partition_table.finish_mark not exists for 14-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sorter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 53-th check, retry later table partition_table.finish_mark not exists for 15-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 54-th check, retry later check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:03:25 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3637536377.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:03:27 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> table ddl_manager.finish_mark not exists for 55-th check, retry later check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.72.250' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:03:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/18427ccf-fa07-4f2d-92a8-2459cb276f95 {ClusterID:default ID:default/18427ccf-fa07-4f2d-92a8-2459cb276f95 Info:{"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} Status:0xc00319ae58 TaskPositions:map[5362f493-5255-4eff-94b2-bf04001e9e84:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 97, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 84, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 87, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 86, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 92, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 90, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 91, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 94, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 95, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 83, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 85, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 101, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5362f493-5255-4eff-94b2-bf04001e9e84 {"id":"5362f493-5255-4eff-94b2-bf04001e9e84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2533b41 5362f493-5255-4eff-94b2-bf04001e9e84 /tidb/cdc/default/default/changefeed/info/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} /tidb/cdc/default/default/changefeed/status/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":450139574314729476,"min-table-barrier-ts":450139575625449476,"admin-job-type":0} /tidb/cdc/default/default/task/position/5362f493-5255-4eff-94b2-bf04001e9e84/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/18427ccf-fa07-4f2d-92a8-2459cb276f95 {ClusterID:default ID:default/18427ccf-fa07-4f2d-92a8-2459cb276f95 Info:{"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} Status:0xc00319ae58 TaskPositions:map[5362f493-5255-4eff-94b2-bf04001e9e84:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 97, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 84, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 87, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 86, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 92, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 90, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 91, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 94, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 95, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 83, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 85, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 101, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5362f493-5255-4eff-94b2-bf04001e9e84 {"id":"5362f493-5255-4eff-94b2-bf04001e9e84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2533b41 5362f493-5255-4eff-94b2-bf04001e9e84 /tidb/cdc/default/default/changefeed/info/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} /tidb/cdc/default/default/changefeed/status/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":450139574314729476,"min-table-barrier-ts":450139575625449476,"admin-job-type":0} /tidb/cdc/default/default/task/position/5362f493-5255-4eff-94b2-bf04001e9e84/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/18427ccf-fa07-4f2d-92a8-2459cb276f95 {ClusterID:default ID:default/18427ccf-fa07-4f2d-92a8-2459cb276f95 Info:{"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} Status:0xc00319ae58 TaskPositions:map[5362f493-5255-4eff-94b2-bf04001e9e84:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 97, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 84, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 87, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 86, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 92, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 90, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 91, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 94, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 95, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 83, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 85, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating tableID: 101, resolvedTs: 450139574314729476, checkpointTs: 450139574314729476, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5362f493-5255-4eff-94b2-bf04001e9e84 {"id":"5362f493-5255-4eff-94b2-bf04001e9e84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde2533b41 5362f493-5255-4eff-94b2-bf04001e9e84 /tidb/cdc/default/default/changefeed/info/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"upstream-id":7375086218234168810,"namespace":"default","changefeed-id":"18427ccf-fa07-4f2d-92a8-2459cb276f95","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:02:06.27895331+08:00","start-ts":450139554103689220,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139554116534280} /tidb/cdc/default/default/changefeed/status/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":450139574314729476,"min-table-barrier-ts":450139575625449476,"admin-job-type":0} /tidb/cdc/default/default/task/position/5362f493-5255-4eff-94b2-bf04001e9e84/18427ccf-fa07-4f2d-92a8-2459cb276f95 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086218234168810 {"id":7375086218234168810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.72.250' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc380000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:19062, start at 2024-05-31 17:03:05.966450551 +0800 CST m=+4.676971598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 56-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.72.250' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 57-th check, retry later check diff successfully [Fri May 31 17:03:32 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f64900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39759, start at 2024-05-31 17:03:32.671601356 +0800 CST m=+4.387623107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.679 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd3a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25159, start at 2024-05-31 17:02:50.647224993 +0800 CST m=+4.547636960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378cd5240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:25243, start at 2024-05-31 17:02:50.704768663 +0800 CST m=+4.550382704 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:04:50.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:02:50.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:52:50.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.29097.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 58-th check, retry later + set +x + tso='450139577324142593 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139577324142593 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 59-th check, retry later Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 60-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa81c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43171, start at 2024-05-31 17:03:37.016986253 +0800 CST m=+4.442051473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sorter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sorter/tiflash/log/error.log \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } table ddl_manager.finish_mark not exists for 61-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] // node [Pipeline] } Starting CDC state checker... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 62-th check, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 64-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 65-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc1500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:18999, start at 2024-05-31 17:03:05.856169062 +0800 CST m=+4.629399242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378dc380000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:19062, start at 2024-05-31 17:03:05.966450551 +0800 CST m=+4.676971598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:05.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:05.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:05.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.23021.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139581225893889 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139581225893889 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:03:49 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.2306623068.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:03:43.306 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 21"] [2024/05/31 17:03:48.308 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 29"] table ddl_manager.finish_mark not exists for 66-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 67-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:03:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf5bc152-92ee-4479-9156-d58c953e2261 {"id":"cf5bc152-92ee-4479-9156-d58c953e2261","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3f4cd8e cf5bc152-92ee-4479-9156-d58c953e2261 /tidb/cdc/default/default/upstream/7375086669870049903 {"id":7375086669870049903,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf5bc152-92ee-4479-9156-d58c953e2261 {"id":"cf5bc152-92ee-4479-9156-d58c953e2261","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3f4cd8e cf5bc152-92ee-4479-9156-d58c953e2261 /tidb/cdc/default/default/upstream/7375086669870049903 {"id":7375086669870049903,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf5bc152-92ee-4479-9156-d58c953e2261 {"id":"cf5bc152-92ee-4479-9156-d58c953e2261","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3f4cd8e cf5bc152-92ee-4479-9156-d58c953e2261 /tidb/cdc/default/default/upstream/7375086669870049903 {"id":7375086669870049903,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.23125.out cli changefeed create --start-ts=450139581225893889 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Create changefeed successfully! ID: a8828812-4e55-43c6-ab44-aa9471b84de7 Info: {"upstream_id":7375086669870049903,"namespace":"default","id":"a8828812-4e55-43c6-ab44-aa9471b84de7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:03:53.384753306+08:00","start_ts":450139581225893889,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139581225893889,"checkpoint_ts":450139581225893889,"checkpoint_time":"2024-05-31 17:03:49.652"} PASS Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table ddl_manager.finish_mark not exists for 68-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.72.250' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null }' + echo '{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null }' { "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null } ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450139580374450182, '"checkpoint_time":' '"2024-05-31' '17:03:46.404",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450139580374450182, '"checkpoint_time":' '"2024-05-31' '17:03:46.404",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null }' + echo '{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null }' { "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139580374450182, "checkpoint_time": "2024-05-31 17:03:46.404", "error": null } ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450139580374450182, '"checkpoint_time":' '"2024-05-31' '17:03:46.404",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450139580374450182, '"checkpoint_time":' '"2024-05-31' '17:03:46.404",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139579063468039, "checkpoint_time": "2024-05-31 17:03:41.403", "error": { "time": "2024-05-31T17:03:42.558129952+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139579155218434]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139579063468039, "checkpoint_time": "2024-05-31 17:03:41.403", "error": { "time": "2024-05-31T17:03:42.558129952+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139579155218434]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7375086329148012408, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139579063468039, "checkpoint_time": "2024-05-31 17:03:41.403", "error": { "time": "2024-05-31T17:03:42.558129952+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139579155218434]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450139579063468039, '"checkpoint_time":' '"2024-05-31' '17:03:41.403",' '"error":' '{' '"time":' '"2024-05-31T17:03:42.558129952+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450139579155218434]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375086329148012408, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450139579063468039, '"checkpoint_time":' '"2024-05-31' '17:03:41.403",' '"error":' '{' '"time":' '"2024-05-31T17:03:42.558129952+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450139579155218434]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139579155218434]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:03:41.753 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139579155218434. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139579155218434]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:03:48 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT table ddl_manager.finish_mark not exists for 69-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f64900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39759, start at 2024-05-31 17:03:32.671601356 +0800 CST m=+4.387623107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.679 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Fri May 31 17:03:57 CST 2024] <<<<<< run test case multi_topics success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 70-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee540016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42981, start at 2024-05-31 17:03:57.88787892 +0800 CST m=+4.448289133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log table ddl_manager.finish_mark not exists for 71-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_with_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa81c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43171, start at 2024-05-31 17:03:37.016986253 +0800 CST m=+4.442051473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... table ddl_manager.finish_mark not exists for 72-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 73-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 74-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ***************** properties ***************** "readproportion"="0" "insertproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "workload"="core" "mysql.port"="4000" "recordcount"="50" "operationcount"="0" "updateproportion"="0" "threadcount"="4" "requestdistribution"="uniform" "mysql.db"="changefeed_reconstruct" "readallfields"="true" ********************************************** Run finished, takes 21.982983ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3031.6, Avg(us): 1739, Min(us): 1108, Max(us): 5825, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table partition_table.t2 not exists for 2-th check, retry later table partition_table.t2 not exists for 3-th check, retry later table partition_table.t2 not exists for 4-th check, retry later table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 75-th check, retry later table partition_table.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 76-th check, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (46256) - No such process wait process 46256 exit for 1-th time... process 46256 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 17:04:11 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.4662546627.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark exists start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.72.250' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:04:13 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> table partition_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:04:14 CST 2024] <<<<<< run test case partition_table success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {ClusterID:default ID:default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 Info:{"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} Status:0xc00084b590 TaskPositions:map[8bdddac5-7204-425f-a7a3-dea057435bc4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139587435823110, checkpointTs: 450139587435823108, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bdddac5-7204-425f-a7a3-dea057435bc4 {"id":"8bdddac5-7204-425f-a7a3-dea057435bc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553fac 8bdddac5-7204-425f-a7a3-dea057435bc4 /tidb/cdc/default/default/changefeed/info/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} /tidb/cdc/default/default/changefeed/status/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":450139587435823108,"min-table-barrier-ts":450139587435823108,"admin-job-type":0} /tidb/cdc/default/default/task/position/8bdddac5-7204-425f-a7a3-dea057435bc4/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086507879429105 {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {ClusterID:default ID:default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 Info:{"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} Status:0xc00084b590 TaskPositions:map[8bdddac5-7204-425f-a7a3-dea057435bc4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139587435823110, checkpointTs: 450139587435823108, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bdddac5-7204-425f-a7a3-dea057435bc4 {"id":"8bdddac5-7204-425f-a7a3-dea057435bc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553fac 8bdddac5-7204-425f-a7a3-dea057435bc4 /tidb/cdc/default/default/changefeed/info/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} /tidb/cdc/default/default/changefeed/status/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":450139587435823108,"min-table-barrier-ts":450139587435823108,"admin-job-type":0} /tidb/cdc/default/default/task/position/8bdddac5-7204-425f-a7a3-dea057435bc4/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086507879429105 {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {ClusterID:default ID:default/5366dbc7-faae-4f20-9fbf-56cccd9a1680 Info:{"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} Status:0xc00084b590 TaskPositions:map[8bdddac5-7204-425f-a7a3-dea057435bc4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139587435823110, checkpointTs: 450139587435823108, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bdddac5-7204-425f-a7a3-dea057435bc4 {"id":"8bdddac5-7204-425f-a7a3-dea057435bc4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3553fac 8bdddac5-7204-425f-a7a3-dea057435bc4 /tidb/cdc/default/default/changefeed/info/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"upstream-id":7375086507879429105,"namespace":"default","changefeed-id":"5366dbc7-faae-4f20-9fbf-56cccd9a1680","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:03:11.44012776+08:00","start-ts":450139571170050057,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139571196002311} /tidb/cdc/default/default/changefeed/status/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":450139587435823108,"min-table-barrier-ts":450139587435823108,"admin-job-type":0} /tidb/cdc/default/default/task/position/8bdddac5-7204-425f-a7a3-dea057435bc4/5366dbc7-faae-4f20-9fbf-56cccd9a1680 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375086507879429105 + grep -q 'etcd info' {"id":7375086507879429105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id "id": "8bdddac5-7204-425f-a7a3-dea057435bc4", "cluster-id": "default" run task successfully Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791d95c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25278, start at 2024-05-31 17:04:12.931308832 +0800 CST m=+4.267060373 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:12.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f648c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39662, start at 2024-05-31 17:03:32.67421258 +0800 CST m=+4.442094900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378f64900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--42853, pid:39759, start at 2024-05-31 17:03:32.671601356 +0800 CST m=+4.387623107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:32.679 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:32.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:32.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) capture_id: 8bdddac5-7204-425f-a7a3-dea057435bc4 check_processor_table_count http://127.0.0.1:2379 5366dbc7-faae-4f20-9fbf-56cccd9a1680 8bdddac5-7204-425f-a7a3-dea057435bc4 1 run task successfully ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=43923 + echo 'proxy port: 10509' proxy port: 10509 + echo 'proxy pid: 43923' proxy pid: 43923 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=10509 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f check_processor_table_count http://127.0.0.1:2379 5366dbc7-faae-4f20-9fbf-56cccd9a1680 8bdddac5-7204-425f-a7a3-dea057435bc4 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later time="2024-05-31T17:04:16+08:00" level=info msg="Listening on 127.0.0.1:10509" time="2024-05-31T17:04:16+08:00" level=info msg="Not intercepting TLS connections" Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_processor_table_count http://127.0.0.1:2379 5366dbc7-faae-4f20-9fbf-56cccd9a1680 8bdddac5-7204-425f-a7a3-dea057435bc4 0 run task successfully ***************** properties ***************** "requestdistribution"="uniform" "readproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.port"="4000" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "recordcount"="50" "updateproportion"="0" "insertproportion"="0" "readallfields"="true" "workload"="core" "scanproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 20.639105ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3242.9, Avg(us): 1630, Min(us): 990, Max(us): 5799, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table changefeed_reconstruct.usertable exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43078, start at 2024-05-31 17:03:37.014642366 +0800 CST m=+4.549138282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f378fa81c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:43171, start at 2024-05-31 17:03:37.016986253 +0800 CST m=+4.442051473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:37.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:37.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:37.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 1-th time, retry later + export http_proxy=http://127.0.0.1:10509 + http_proxy=http://127.0.0.1:10509 + export https_proxy=http://127.0.0.1:10509 + https_proxy=http://127.0.0.1:10509 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.72.250:2379' try to connect pd cluster via proxy, pd addr: 10.233.72.250:2379 + ensure 10 curl http://10.233.72.250:2379/ curl http://10.233.72.250:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 15886 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 43923 started proxy pid: 43923 + echo started proxy at port: 10509 started proxy at port: 10509 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.72.250 2379 + pd_host=10.233.72.250 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://10.233.72.250:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.44560.out cli tso query --pd=http://10.233.72.250:2379 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed test leveldb sorter + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.47246.out cli tso query --pd=http://127.0.0.1:2379 failed proxying s2c rpc error: code = Unavailable desc = client disconnected + set +x + tso='450139589302550529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139589302550529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139589302550529 + echo 'query start ts: 450139589302550529' query start ts: 450139589302550529 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Fri May 31 17:04:20 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.4460844610.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 10509 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 10509 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Fri, 31 May 2024 09:04:20 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139589361532931 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139589361532931 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:04:22 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee540016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42981, start at 2024-05-31 17:03:57.88787892 +0800 CST m=+4.448289133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Fri May 31 17:04:13 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.4937149373.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d90ee026-1a61-40e2-a5b2-71707cd287a0 {"id":"d90ee026-1a61-40e2-a5b2-71707cd287a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde36cd010 d90ee026-1a61-40e2-a5b2-71707cd287a0 /tidb/cdc/default/default/upstream/7375086534570845930 {"id":7375086534570845930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d90ee026-1a61-40e2-a5b2-71707cd287a0 {"id":"d90ee026-1a61-40e2-a5b2-71707cd287a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde36cd010 d90ee026-1a61-40e2-a5b2-71707cd287a0 /tidb/cdc/default/default/upstream/7375086534570845930 {"id":7375086534570845930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d90ee026-1a61-40e2-a5b2-71707cd287a0 {"id":"d90ee026-1a61-40e2-a5b2-71707cd287a0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde36cd010 d90ee026-1a61-40e2-a5b2-71707cd287a0 /tidb/cdc/default/default/upstream/7375086534570845930 {"id":7375086534570845930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.49427.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: fcf9bd64-f046-48b2-bf5c-4f0869d2b19f Info: {"upstream_id":7375086534570845930,"namespace":"default","id":"fcf9bd64-f046-48b2-bf5c-4f0869d2b19f","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-31T17:04:17.337490155+08:00","start_ts":450139588453466113,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139588453466113,"checkpoint_ts":450139588453466113,"checkpoint_time":"2024-05-31 17:04:17.223"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 [2024/05/31 17:04:18.157 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:18.158 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:18.159 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq0] [2024/05/31 17:04:18.239 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.247 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.099718ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.247 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq1] [2024/05/31 17:04:18.328 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.336 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.076211ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.336 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq2] [2024/05/31 17:04:18.418 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.426 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.526711ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.426 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq3] [2024/05/31 17:04:18.506 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.515 +08:00] [INFO] [case.go:364] ["insert 100 takes 9.442628ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.516 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq4] [2024/05/31 17:04:18.593 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.600 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.17429ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/31 17:04:18.601 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts0] [2024/05/31 17:04:18.686 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.431538ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/31 17:04:18.687 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts1] [2024/05/31 17:04:18.773 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.2852ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/31 17:04:18.773 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts2] [2024/05/31 17:04:18.864 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.950498ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/31 17:04:18.864 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts3] [2024/05/31 17:04:18.954 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.395355ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/31 17:04:18.954 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts4] [2024/05/31 17:04:19.045 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.773961ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/31 17:04:21.126 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:23.135 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 10509 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 10509 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 617 < Content-Type: text/plain; charset=utf-8 < Date: Fri, 31 May 2024 09:04:23 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/852458ce-60b7-4ba5-8afb-32470ec39797 {"id":"852458ce-60b7-4ba5-8afb-32470ec39797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/44048fcde45d3192 852458ce-60b7-4ba5-8afb-32470ec39797 /tidb/cdc/default/default/upstream/7375086796508822161 {"id":7375086796508822161,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.72.250:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/852458ce-60b7-4ba5-8afb-32470ec39797 {"id":"852458ce-60b7-4ba5-8afb-32470ec39797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/44048fcde45d3192 852458ce-60b7-4ba5-8afb-32470ec39797 /tidb/cdc/default/default/upstream/7375086796508822161 {"id":7375086796508822161,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.72.250:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/852458ce-60b7-4ba5-8afb-32470ec39797 {"id":"852458ce-60b7-4ba5-8afb-32470ec39797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/44048fcde45d3192 852458ce-60b7-4ba5-8afb-32470ec39797 /tidb/cdc/default/default/upstream/7375086796508822161 {"id":7375086796508822161,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.72.250:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=450139589302550529 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.44679.out cli changefeed create --start-ts=450139589302550529 --sink-uri=blackhole:/// failed proxying s2c rpc error: code = Canceled desc = context canceled Create changefeed successfully! ID: 322e5d17-903e-42f4-a420-f57e861aed0a Info: {"upstream_id":7375086796508822161,"namespace":"default","id":"322e5d17-903e-42f4-a420-f57e861aed0a","sink_uri":"blackhole:///","create_time":"2024-05-31T17:04:24.222036946+08:00","start_ts":450139589302550529,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139589302550529,"checkpoint_ts":450139589302550529,"checkpoint_time":"2024-05-31 17:04:20.462"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 5 start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:04:25.145 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:04:27.156 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:04:28.839 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 4"] [2024/05/31 17:04:29.165 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq + service_type_count=6 + echo 'captured services: ' captured services: + echo . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv '[2024/05' . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv [2024/05 + '[' 6 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Fri May 31 17:04:29 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Fri May 31 17:04:29 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/31 17:04:31.180 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:04:33.188 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379314bc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31732, start at 2024-05-31 17:04:33.082602052 +0800 CST m=+4.200699736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:04:35.197 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791d95c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25278, start at 2024-05-31 17:04:12.931308832 +0800 CST m=+4.267060373 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:12.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/05/31 17:04:37.206 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379342dc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:29013, start at 2024-05-31 17:04:36.027148271 +0800 CST m=+4.285240509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:36.034 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:36.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:36.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:04:39.216 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:39.225 +08:00] [INFO] [case.go:638] ["rows scan failed"] [error="sql: Scan error on column index 0, name \"END_TIME\": converting NULL to string is unsupported"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:04:41.226 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:41.235 +08:00] [INFO] [case.go:496] ["all tables synced"] [endTs="2024-05-31 17:04:39"] + stop_proxy + kill 43923 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Fri May 31 17:04:41 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42895, start at 2024-05-31 17:03:57.888084774 +0800 CST m=+4.543084630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.899 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.899 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3790ee540016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:42981, start at 2024-05-31 17:03:57.88787892 +0800 CST m=+4.448289133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:05:57.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:03:57.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:53:57.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed ***************** properties ***************** "requestdistribution"="uniform" "insertproportion"="0" "workload"="core" "operationcount"="0" "dotransactions"="false" "updateproportion"="0" "mysql.db"="multi_capture_1" "readallfields"="true" "mysql.host"="127.0.0.1" "recordcount"="10" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "threadcount"="2" "scanproportion"="0" ********************************************** Run finished, takes 11.380192ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1720.2, Avg(us): 2158, Min(us): 1146, Max(us): 5603, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.user"="root" "threadcount"="2" "scanproportion"="0" "requestdistribution"="uniform" "mysql.db"="multi_capture_2" "dotransactions"="false" "readproportion"="0" "insertproportion"="0" "recordcount"="10" "mysql.port"="4000" "updateproportion"="0" "operationcount"="0" "readallfields"="true" "workload"="core" ********************************************** Run finished, takes 11.690588ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1636.5, Avg(us): 2255, Min(us): 1275, Max(us): 5436, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "scanproportion"="0" "updateproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10" "mysql.db"="multi_capture_3" "mysql.user"="root" "readallfields"="true" "workload"="core" "operationcount"="0" "mysql.port"="4000" "dotransactions"="false" ********************************************** Run finished, takes 10.110095ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1946.5, Avg(us): 1945, Min(us): 959, Max(us): 4902, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readallfields"="true" "threadcount"="2" "mysql.db"="multi_capture_4" "insertproportion"="0" "mysql.user"="root" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="10" "updateproportion"="0" "readproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 10.653532ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1802.0, Avg(us): 2055, Min(us): 1136, Max(us): 5026, 95th(us): 6000, 99th(us): 6000 [Fri May 31 17:04:35 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.2928329285.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:04:38 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.2934629348.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:41 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:04:41 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.2941029412.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri May 31 17:04:42 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.4713247134.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:04:43.930 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/31 17:04:43.930 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark1000] \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:45 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/05f27fa2-1d91-4469-839a-5cb89bfe161a {"id":"05f27fa2-1d91-4469-839a-5cb89bfe161a","address":"127.0.0.1:8303","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8908b 05f27fa2-1d91-4469-839a-5cb89bfe161a /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/05f27fa2-1d91-4469-839a-5cb89bfe161a {"id":"05f27fa2-1d91-4469-839a-5cb89bfe161a","address":"127.0.0.1:8303","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8908b 05f27fa2-1d91-4469-839a-5cb89bfe161a /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/040aac64-ec5e-40ae-a748-9e6f079131c4 {"id":"040aac64-ec5e-40ae-a748-9e6f079131c4","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/05f27fa2-1d91-4469-839a-5cb89bfe161a {"id":"05f27fa2-1d91-4469-839a-5cb89bfe161a","address":"127.0.0.1:8303","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/e6e00aa3-de4f-44b9-96ed-57a16112f804 {"id":"e6e00aa3-de4f-44b9-96ed-57a16112f804","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8905b 040aac64-ec5e-40ae-a748-9e6f079131c4 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8907d e6e00aa3-de4f-44b9-96ed-57a16112f804 /tidb/cdc/default/__cdc_meta__/owner/22318fcde3b8908b 05f27fa2-1d91-4469-839a-5cb89bfe161a /tidb/cdc/default/default/upstream/7375086609152026653 {"id":7375086609152026653,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.29478.out cli changefeed create --start-ts=450139577324142593 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: ceff10a3-8c68-4080-8e8d-cc22756b9e06 Info: {"upstream_id":7375086609152026653,"namespace":"default","id":"ceff10a3-8c68-4080-8e8d-cc22756b9e06","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:04:45.346199341+08:00","start_ts":450139577324142593,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139577324142593,"checkpoint_ts":450139577324142593,"checkpoint_time":"2024-05-31 17:03:34.768"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table multi_capture_1.usertable not exists for 1-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [2024/05/31 17:04:45.930 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:45.940 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark1000] [2024/05/31 17:04:45.940 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:04:44"] [2024/05/31 17:04:45.944 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:45.947 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:45.952 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:45.956 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 {"id":"b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4c1038e b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 /tidb/cdc/default/default/upstream/7375086902124195218 {"id":7375086902124195218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 {"id":"b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4c1038e b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 /tidb/cdc/default/default/upstream/7375086902124195218 {"id":7375086902124195218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 {"id":"b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4c1038e b0e1cf15-d2f0-4bc7-9435-dd07ef92eb35 /tidb/cdc/default/default/upstream/7375086902124195218 {"id":7375086902124195218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/31 17:04:46.026 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-normal Info: {"upstream_id":7375086902124195218,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-18505?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-31T17:04:46.117693113+08:00","start_ts":450139595161993217,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139595161993217,"checkpoint_ts":450139595161993217,"checkpoint_time":"2024-05-31 17:04:42.814"} [2024/05/31 17:04:46.209 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7375086902124195218,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25678?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-31T17:04:46.267659056+08:00","start_ts":450139595161993217,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139595161993217,"checkpoint_ts":450139595161993217,"checkpoint_time":"2024-05-31 17:04:42.814"} [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/31 17:04:46.358 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7375086902124195218,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-18001?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-31T17:04:46.417078982+08:00","start_ts":450139595161993217,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139595161993217,"checkpoint_ts":450139595161993217,"checkpoint_time":"2024-05-31 17:04:42.814"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-18505?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-18505?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25678?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-25678?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-18001?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri May 31 17:04:46 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-18001?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793df500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:48055, start at 2024-05-31 17:04:46.070854757 +0800 CST m=+5.089536143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:46.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:46.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:46.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // withEnv [2024/05/31 17:04:46.985 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/31 17:04:46.985 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark2000] [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] // stage Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later [2024/05/31 17:04:49.017 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:49.028 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark2000] [2024/05/31 17:04:49.028 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:04:48"] [2024/05/31 17:04:49.031 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:49.034 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:49.038 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:49.042 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting CDC state checker... table drop_tables.c not exists for 1-th check, retry later table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later table drop_tables.c not exists for 2-th check, retry later [2024/05/31 17:04:50.128 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/31 17:04:50.128 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark3000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later [2024/05/31 17:04:52.129 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:52.138 +08:00] [INFO] [case.go:638] ["rows scan failed"] [error="sql: Scan error on column index 0, name \"END_TIME\": converting NULL to string is unsupported"] [2024/05/31 17:04:52.329 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/31 17:04:52.329 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark4000] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 3-th check, retry later table multi_capture_4.usertable exists check diff failed 1-th time, retry later [2024/05/31 17:04:54.138 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:54.227 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark3000] [2024/05/31 17:04:54.227 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:04:52"] [2024/05/31 17:04:54.230 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:54.234 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:54.239 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:54.320 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:54.333 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791db100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25198, start at 2024-05-31 17:04:13.022943015 +0800 CST m=+4.437709525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:13.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3791d95c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:25278, start at 2024-05-31 17:04:12.931308832 +0800 CST m=+4.267060373 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:12.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:12.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:12.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table drop_tables.c not exists for 4-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:04:54 CST 2024] <<<<<< START cdc server in new_ci_collation_with_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_with_old_value.2920329205.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:04:55.427 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/31 17:04:55.427 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark5000] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ***************** properties ***************** "dotransactions"="false" "recordcount"="20" "requestdistribution"="uniform" "updateproportion"="0" "readallfields"="true" "mysql.user"="root" "readproportion"="0" "insertproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="multi_capture_1" "threadcount"="2" "scanproportion"="0" "workload"="core" ********************************************** Run finished, takes 10.643049ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2138.5, Avg(us): 1013, Min(us): 566, Max(us): 1577, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "operationcount"="0" "updateproportion"="0" "recordcount"="20" "scanproportion"="0" "mysql.db"="multi_capture_2" "readallfields"="true" "readproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "threadcount"="2" "workload"="core" "mysql.user"="root" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 9.767873ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2316.9, Avg(us): 922, Min(us): 479, Max(us): 1587, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "readallfields"="true" "mysql.user"="root" "workload"="core" "readproportion"="0" "scanproportion"="0" "recordcount"="20" "operationcount"="0" "dotransactions"="false" "requestdistribution"="uniform" "mysql.port"="4000" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_3" "updateproportion"="0" ********************************************** Run finished, takes 9.881821ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2291.3, Avg(us): 931, Min(us): 511, Max(us): 1558, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "readproportion"="0" "readallfields"="true" "insertproportion"="0" "workload"="core" "threadcount"="2" "updateproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="20" "mysql.db"="multi_capture_4" "dotransactions"="false" "operationcount"="0" ********************************************** Run finished, takes 10.342737ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2134.8, Avg(us): 896, Min(us): 469, Max(us): 1475, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later [2024/05/31 17:04:56.347 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:56.434 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark4000] [2024/05/31 17:04:56.434 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:04:55"] [2024/05/31 17:04:56.438 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:56.519 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:56.524 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:56.529 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table drop_tables.c not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:04:57.427 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379314bc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31732, start at 2024-05-31 17:04:33.082602052 +0800 CST m=+4.200699736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:04:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd825027-e1ba-4470-835e-7a39edc7355a {"id":"dd825027-e1ba-4470-835e-7a39edc7355a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4f7848e dd825027-e1ba-4470-835e-7a39edc7355a /tidb/cdc/default/default/upstream/7375086966361652239 {"id":7375086966361652239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd825027-e1ba-4470-835e-7a39edc7355a {"id":"dd825027-e1ba-4470-835e-7a39edc7355a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4f7848e dd825027-e1ba-4470-835e-7a39edc7355a /tidb/cdc/default/default/upstream/7375086966361652239 {"id":7375086966361652239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd825027-e1ba-4470-835e-7a39edc7355a {"id":"dd825027-e1ba-4470-835e-7a39edc7355a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde4f7848e dd825027-e1ba-4470-835e-7a39edc7355a /tidb/cdc/default/default/upstream/7375086966361652239 {"id":7375086966361652239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 0db41ed2-471c-4e82-8de3-db931a0b1d34 Info: {"upstream_id":7375086966361652239,"namespace":"default","id":"0db41ed2-471c-4e82-8de3-db931a0b1d34","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-31T17:04:57.961489358+08:00","start_ts":450139598282031105,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139598282031105,"checkpoint_ts":450139598282031105,"checkpoint_time":"2024-05-31 17:04:54.716"} [2024/05/31 17:04:58.450 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/31 17:04:58.450 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark6000] table drop_tables.c exists check diff successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:04:58 CST 2024] <<<<<< run test case multi_capture success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:04:59 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:04:59.517 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:04:59.531 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark5000] [2024/05/31 17:04:59.531 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:04:58"] [2024/05/31 17:04:59.533 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:59.537 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:04:59.541 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:04:59.545 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:00.517 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379342dc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:29013, start at 2024-05-31 17:04:36.027148271 +0800 CST m=+4.285240509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:36.034 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:36.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:36.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4155 0 --:--:-- --:--:-- --:--:-- 4169 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:04:48.928","puller_resolved_ts":"2024-05-31 17:04:42.928","last_synced_ts":"2024-05-31 17:02:32.429","now_ts":"2024-05-31 17:04:49.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:04:48.928","puller_resolved_ts":"2024-05-31' '17:04:42.928","last_synced_ts":"2024-05-31' '17:02:32.429","now_ts":"2024-05-31' '17:04:49.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 24870 11.3 0.0 11444064 147580 ? Sl 17:01 0:23 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 24896 6.3 0.0 11492312 140092 ? Sl 17:01 0:12 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-31T17:04:58.788+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:04:58.792+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:04:58.928+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/31 17:05:02.229 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/31 17:05:02.229 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark7000] [2024/05/31 17:05:02.529 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:02.546 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark6000] [2024/05/31 17:05:02.546 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:01"] [2024/05/31 17:05:02.619 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:02.629 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:02.633 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:02.637 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:04.229 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] {"level":"warn","ts":"2024-05-31T17:05:04.791+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:04.794+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:04.929+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/31 17:05:05.031 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/31 17:05:05.031 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark8000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:05:06.241 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:06.324 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark7000] [2024/05/31 17:05:06.324 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:04"] [2024/05/31 17:05:06.327 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:06.329 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:06.335 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:06.340 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:07.031 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:05:08.231 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/31 17:05:08.231 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark9000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:05:09.041 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... {"level":"warn","ts":"2024-05-31T17:05:08.767+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:05:08.767+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:05:08.773+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:05:08.773+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:05:08.906+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:05:08.907+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717146308.9870172,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001684fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717146308.987118,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717146309.0111232,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014ab6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717146309.0112317,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717146309.102908,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001765180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717146309.1030252,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [2024/05/31 17:05:09.118 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark8000] [2024/05/31 17:05:09.118 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:07"] [2024/05/31 17:05:09.121 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:09.125 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:09.130 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:09.134 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793df500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:48055, start at 2024-05-31 17:04:46.070854757 +0800 CST m=+5.089536143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:46.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:46.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:46.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 17:05:10.231 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:05:10.792+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:10.794+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... {"level":"warn","ts":"2024-05-31T17:05:10.929+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:05:11.752 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/31 17:05:11.752 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark10000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:05:12.241 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:12.252 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark9000] [2024/05/31 17:05:12.252 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:10"] [2024/05/31 17:05:12.321 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:12.325 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:12.332 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:12.337 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/31 17:05:13.753 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:13.831 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark10000] [2024/05/31 17:05:13.832 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:13"] [2024/05/31 17:05:13.918 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:13.921 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:13.925 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:13.930 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:05:15.030 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/31 17:05:15.030 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark11000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379313580020 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31644, start at 2024-05-31 17:04:33.025226989 +0800 CST m=+4.204791141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379314bc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:31732, start at 2024-05-31 17:04:33.082602052 +0800 CST m=+4.200699736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:33.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:33.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:33.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:05:16 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3577635778.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:05:16.793+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:16.795+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:16.930+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/31 17:05:17.030 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:17.229 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/31 17:05:17.229 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark12000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "recordcount"="3000" "workload"="core" "threadcount"="10" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "mysql.db"="sorter" "mysql.host"="127.0.0.1" "readallfields"="true" "operationcount"="0" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "mysql.user"="root" "updateproportion"="0" ********************************************** Run finished, takes 587.612909ms INSERT - Takes(s): 0.6, Count: 3000, OPS: 5152.6, Avg(us): 1915, Min(us): 872, Max(us): 43965, 95th(us): 2000, 99th(us): 40000 [Fri May 31 17:05:17 CST 2024] <<<<<< START cdc server in sorter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.4739647398.out server --log-file /tmp/tidb_cdc_test/sorter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sorter/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:05:19.045 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:19.125 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark11000] [2024/05/31 17:05:19.125 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:17"] [2024/05/31 17:05:19.128 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:19.131 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:19.136 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:19.140 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379341580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:28925, start at 2024-05-31 17:04:35.970548267 +0800 CST m=+4.307329015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:35.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:35.976 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:35.976 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379342dc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:29013, start at 2024-05-31 17:04:36.027148271 +0800 CST m=+4.285240509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:36.034 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:36.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:36.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + (( i++ )) + (( i <= 50 )) [2024/05/31 17:05:19.229 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:05:19 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf26c721-1fee-45aa-9def-eae55d33939b {"id":"cf26c721-1fee-45aa-9def-eae55d33939b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548d98e cf26c721-1fee-45aa-9def-eae55d33939b /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf26c721-1fee-45aa-9def-eae55d33939b {"id":"cf26c721-1fee-45aa-9def-eae55d33939b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548d98e cf26c721-1fee-45aa-9def-eae55d33939b /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf26c721-1fee-45aa-9def-eae55d33939b {"id":"cf26c721-1fee-45aa-9def-eae55d33939b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548d98e cf26c721-1fee-45aa-9def-eae55d33939b /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.35850.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7375087052477624293,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-31T17:05:19.453294718+08:00","start_ts":450139604736802820,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139604736802820,"checkpoint_ts":450139604736802820,"checkpoint_time":"2024-05-31 17:05:19.339"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:05:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd {"id":"13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde46bbe11 13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd /tidb/cdc/default/default/upstream/7375086812389630425 {"id":7375086812389630425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd {"id":"13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde46bbe11 13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd /tidb/cdc/default/default/upstream/7375086812389630425 {"id":7375086812389630425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd {"id":"13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde46bbe11 13a9bc9e-1027-4200-9f5b-94f9a6a3ecbd /tidb/cdc/default/default/upstream/7375086812389630425 {"id":7375086812389630425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.47458.out cli changefeed create -c ticdc-leveldb-sorter-test-31991 --start-ts=450139589361532931 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' --sort-engine=unified Create changefeed successfully! ID: ticdc-leveldb-sorter-test-31991 Info: {"upstream_id":7375086812389630425,"namespace":"default","id":"ticdc-leveldb-sorter-test-31991","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:05:20.644458279+08:00","start_ts":450139589361532931,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139589361532931,"checkpoint_ts":450139589361532931,"checkpoint_time":"2024-05-31 17:04:20.687"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table sorter.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e450001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30696, start at 2024-05-31 17:05:19.16853234 +0800 CST m=+4.300127352 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log [2024/05/31 17:05:21.031 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/31 17:05:21.031 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark13000] [2024/05/31 17:05:21.243 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:21.321 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark12000] [2024/05/31 17:05:21.321 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:19"] [2024/05/31 17:05:21.329 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:21.333 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:21.337 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:21.340 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:21.424 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:21.520 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:21.722 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:22.020 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795fa740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24255, start at 2024-05-31 17:05:20.579151566 +0800 CST m=+4.413808404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log [2024/05/31 17:05:22.366 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:22.457 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:05:22.795+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:22.796+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:22.931+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/31 17:05:23.031 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:23.042 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark13000] [2024/05/31 17:05:23.042 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:22"] [2024/05/31 17:05:23.119 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:23.122 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:23.127 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:23.132 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:23.220 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:23.320 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] table sorter.usertable exists table sorter.check1 not exists for 1-th check, retry later Starting CDC state checker... [2024/05/31 17:05:23.521 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:23.831 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:23.920 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 17:05:24.331 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark14000] [2024/05/31 17:05:24.331 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark14000] [2024/05/31 17:05:24.721 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] Starting CDC state checker... [2024/05/31 17:05:25.221 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] table sorter.check1 not exists for 2-th check, retry later [2024/05/31 17:05:17.170 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.627927794s"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/05/31 17:05:26.121 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:26.331 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:26.341 +08:00] [INFO] [case.go:638] ["rows scan failed"] [error="sql: Scan error on column index 0, name \"END_TIME\": converting NULL to string is unsupported"] [2024/05/31 17:05:26.723 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts3 expect 100000, but got 99944"] [errorVerbose="verify balance failed, accounts3 expect 100000, but got 99944\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:26.734 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/31 17:05:26.734 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark15000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table sorter.check1 exists check diff successfully check diff failed 1-th time, retry later [2024/05/31 17:05:28.341 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/05/31 17:05:28.425 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark14000] [2024/05/31 17:05:28.425 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-05-31 17:05:26"] [2024/05/31 17:05:28.429 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:28.431 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:28.435 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/31 17:05:28.440 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/31 17:05:28.725 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts2 expect 100000, but got 99807"] [errorVerbose="verify balance failed, accounts2 expect 100000, but got 99807\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/05/31 17:05:28.725 +08:00] [PANIC] [case.go:595] ["downstream verify failed"] [error="[CDC:ErrReachMaxTry]reach maximum try: 10, error: verify balance failed, accounts2 expect 100000, but got 99807: verify balance failed, accounts2 expect 100000, but got 99807"] [stack="main.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:595\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] panic: downstream verify failed goroutine 56 [running]: go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc0002a1e40?}) /go/pkg/mod/go.uber.org/zap@v1.23.0/zapcore/entry.go:198 +0x65 go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0002644e0, {0xc000363500, 0x1, 0x1}) /go/pkg/mod/go.uber.org/zap@v1.23.0/zapcore/entry.go:264 +0x3ec go.uber.org/zap.(*Logger).Panic(0xc000363500?, {0x8fd2c3?, 0xc0001d40a8?}, {0xc000363500, 0x1, 0x1}) /go/pkg/mod/go.uber.org/zap@v1.23.0/logger.go:251 +0x59 github.com/pingcap/log.Panic({0x8fd2c3?, 0x13?}, {0xc000363500?, 0x9f3408?, 0xc000570060?}) /go/pkg/mod/github.com/pingcap/log@v1.1.1-0.20221116035753-734d527bc87c/global.go:54 +0x8b main.run.func2() /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:595 +0xa85 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75 +0x64 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:72 +0xa5 exit status 2 + stop_tidb_cluster 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:05:28.796+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:28.796+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:28.933+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table sorter.check2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793dd30001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:47958, start at 2024-05-31 17:04:45.9476413 +0800 CST m=+5.032931833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:45.955 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:45.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:45.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3793df500016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:48055, start at 2024-05-31 17:04:46.070854757 +0800 CST m=+5.089536143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:06:46.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:04:46.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:54:46.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:05:31 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.5202652028.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table sorter.check2 not exists for 2-th check, retry later table sorter.check2 exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ***************** properties ***************** "mysql.db"="sorter" "operationcount"="0" "readallfields"="true" "recordcount"="3000" "threadcount"="10" "mysql.host"="127.0.0.1" "updateproportion"="0" "readproportion"="0" "workload"="core" "mysql.port"="4000" "scanproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "insertproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 814.144084ms INSERT - Takes(s): 0.8, Count: 3000, OPS: 3711.9, Avg(us): 2659, Min(us): 933, Max(us): 65429, 95th(us): 3000, 99th(us): 53000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:05:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd9873b-6566-44e5-b6e8-693f201276c5 {"id":"6cd9873b-6566-44e5-b6e8-693f201276c5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d858e 6cd9873b-6566-44e5-b6e8-693f201276c5 /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd9873b-6566-44e5-b6e8-693f201276c5 {"id":"6cd9873b-6566-44e5-b6e8-693f201276c5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d858e 6cd9873b-6566-44e5-b6e8-693f201276c5 /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd9873b-6566-44e5-b6e8-693f201276c5 {"id":"6cd9873b-6566-44e5-b6e8-693f201276c5","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d858e 6cd9873b-6566-44e5-b6e8-693f201276c5 /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table sorter.check3 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:05:34.798+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:34.798+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:34.934+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000994a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/31 17:05:31.283 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 17:05:31.284 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 17:05:31.284 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139574314729476] [resolvedTs=450139607384457218] [2024/05/31 17:05:31.284 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:05:31.285 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:05:31.285 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.103057ms] [2024/05/31 17:05:31.285 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=1.274452ms] [2024/05/31 17:05:31.289 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:05:31.291 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:05:31.292 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 17:05:31.293 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 17:05:31.293 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:05:31.293 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:05:31.297 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:05:31.299 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 17:05:31.299 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450139575861379081,\"CommitTs\":450139575861379085,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/31 17:05:31.299 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:31.501 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:31.702 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:31.902 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:32.103 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:32.304 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:32.505 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:32.706 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:32.906 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.107 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.308 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450139575861379081,\"CommitTs\":450139575861379085,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/31 17:05:33.508 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="alter table t1 drop partition p1"] [duration=199.986168ms] [namespace=] [changefeed=] [2024/05/31 17:05:33.508 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450139575979343880,\"CommitTs\":450139575979343885,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/31 17:05:33.508 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.508 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.508 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.508 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.508 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.709 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450139575979343880,\"CommitTs\":450139575979343885,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/31 17:05:33.854 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [duration=145.319382ms] [namespace=] [changefeed=] [2024/05/31 17:05:33.854 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450139576110415882,\"CommitTs\":450139576123523073,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/31 17:05:33.854 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.854 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.854 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:33.854 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.055 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.055 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.055 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.055 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.256 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.256 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.256 +08:00] [INFO] [redo.go:287] [applyDDL] [2024/05/31 17:05:34.256 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450139576110415882,\"CommitTs\":450139576123523073,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/31 17:05:34.398 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [duration=142.817627ms] [namespace=] [changefeed=] [2024/05/31 17:05:34.399 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.399 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.499 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.499 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.599 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.599 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.699 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.699 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.800 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.800 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.900 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=450139607384457218] [2024/05/31 17:05:34.900 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.001 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.001 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.102 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.102 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.202 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.202 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=450139607384457218] start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [2024/05/31 17:05:35.503 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.503 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.603 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.603 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.905 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=450139607384457218] [2024/05/31 17:05:35.905 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=450139607384457218] [2024/05/31 17:05:36.005 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=47] [appliedDDLCount=3] [currentCheckpoint=450139607384457218] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 17:05:36.005 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] Apply redo log successfully check diff successfully [Fri May 31 17:05:36 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 test table sorter.check3 not exists for 2-th check, retry later Verifying downstream PD is started... table sorter.check3 exists check diff successfully table sorter.usertable2 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release {"level":"warn","ts":"2024-05-31T17:05:40.799+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e31180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:05:40.800+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d76fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit + stop_tidb_cluster [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh + ls /tmp/tidb_cdc_test/ api_v2 autorandom availability bank cov.api_v2.2386123863.out cov.autorandom.3462534627.out cov.availability.4009640098.out cov.availability.4027040272.out cov.availability.4047340475.out cov.availability.4061440616.out cov.availability.4077240774.out cov.availability.4127641278.out cov.availability.4154741549.out cov.availability.4169241694.out cov.availability.4185041852.out cov.availability.4238942391.out cov.availability.4254342545.out cov.availability.4318243184.out cov.availability.4338143383.out cov.availability.4353043532.out cov.availability.4374343745.out cov.availability.4395643958.out cov.availability.cli.39970.out cov.availability.cli.40158.out cov.bank.cli.49427.out sql_res.api_v2.txt sql_res.autorandom.txt sql_res.availability.txt sql_res.bank.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G0.tar.gz /tmp/tidb_cdc_test/autorandom/pd1.log /tmp/tidb_cdc_test/autorandom/tidb_other.log /tmp/tidb_cdc_test/autorandom/tikv3.log /tmp/tidb_cdc_test/autorandom/cdc.log /tmp/tidb_cdc_test/autorandom/sync_diff_inspector.log /tmp/tidb_cdc_test/autorandom/tidb_down.log /tmp/tidb_cdc_test/autorandom/tikv1.log /tmp/tidb_cdc_test/autorandom/tidb-slow.log /tmp/tidb_cdc_test/autorandom/tikv2.log /tmp/tidb_cdc_test/autorandom/down_pd.log /tmp/tidb_cdc_test/autorandom/stdout.log /tmp/tidb_cdc_test/autorandom/tidb.log /tmp/tidb_cdc_test/autorandom/tikv_down.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/tikv2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tikv_down.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/api_v2/pd1.log /tmp/tidb_cdc_test/api_v2/tidb_other.log /tmp/tidb_cdc_test/api_v2/tikv3.log /tmp/tidb_cdc_test/api_v2/cdc.log /tmp/tidb_cdc_test/api_v2/tidb_down.log /tmp/tidb_cdc_test/api_v2/tikv1.log /tmp/tidb_cdc_test/api_v2/tidb-slow.log /tmp/tidb_cdc_test/api_v2/tikv2.log /tmp/tidb_cdc_test/api_v2/down_pd.log /tmp/tidb_cdc_test/api_v2/stdout.log /tmp/tidb_cdc_test/api_v2/tidb.log /tmp/tidb_cdc_test/api_v2/tikv_down.log /tmp/tidb_cdc_test/bank/pd1.log /tmp/tidb_cdc_test/bank/tikv_down/db/000028.log /tmp/tidb_cdc_test/bank/tidb_other.log /tmp/tidb_cdc_test/bank/pd1/region-meta/000005.log /tmp/tidb_cdc_test/bank/pd1/hot-region/000004.log /tmp/tidb_cdc_test/bank/tikv1/db/000018.log /tmp/tidb_cdc_test/bank/tikv1/db/000025.log /tmp/tidb_cdc_test/bank/tiflash/log/server.log /tmp/tidb_cdc_test/bank/tiflash/log/error.log /tmp/tidb_cdc_test/bank/tiflash/log/proxy.log /tmp/tidb_cdc_test/bank/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/bank/down_pd/region-meta/000005.log /tmp/tidb_cdc_test/bank/down_pd/hot-region/000004.log /tmp/tidb_cdc_test/bank/tikv3.log /tmp/tidb_cdc_test/bank/cdc.log /tmp/tidb_cdc_test/bank/tidb_down.log /tmp/tidb_cdc_test/bank/tikv1.log /tmp/tidb_cdc_test/bank/tikv3/db/000018.log /tmp/tidb_cdc_test/bank/tikv3/db/000025.log /tmp/tidb_cdc_test/bank/tikv2/db/000032.log /tmp/tidb_cdc_test/bank/tikv2/db/000028.log /tmp/tidb_cdc_test/bank/tidb-slow.log /tmp/tidb_cdc_test/bank/tikv2.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/bank/tikv2-2024-05-31T17-05-10.419.log /tmp/tidb_cdc_test/bank/down_pd.log /tmp/tidb_cdc_test/bank/stdout.log /tmp/tidb_cdc_test/bank/tidb.log /tmp/tidb_cdc_test/bank/tikv_down.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/autorandom/pd1.log /tmp/tidb_cdc_test/autorandom/tidb_other.log /tmp/tidb_cdc_test/autorandom/tikv3.log /tmp/tidb_cdc_test/autorandom/cdc.log /tmp/tidb_cdc_test/autorandom/sync_diff_inspector.log /tmp/tidb_cdc_test/autorandom/tidb_down.log /tmp/tidb_cdc_test/autorandom/tikv1.log /tmp/tidb_cdc_test/autorandom/tidb-slow.log /tmp/tidb_cdc_test/autorandom/tikv2.log table sorter.usertable2 exists table sorter.check4 not exists for 1-th check, retry later /tmp/tidb_cdc_test/autorandom/down_pd.log /tmp/tidb_cdc_test/autorandom/stdout.log /tmp/tidb_cdc_test/autorandom/tidb.log /tmp/tidb_cdc_test/autorandom/tikv_down.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/tikv2.log Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tikv_down.log Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/api_v2/pd1.log /tmp/tidb_cdc_test/api_v2/tidb_other.log /tmp/tidb_cdc_test/api_v2/tikv3.log /tmp/tidb_cdc_test/api_v2/cdc.log /tmp/tidb_cdc_test/api_v2/tidb_down.log /tmp/tidb_cdc_test/api_v2/tikv1.log /tmp/tidb_cdc_test/api_v2/tidb-slow.log /tmp/tidb_cdc_test/api_v2/tikv2.log /tmp/tidb_cdc_test/api_v2/down_pd.log /tmp/tidb_cdc_test/api_v2/stdout.log /tmp/tidb_cdc_test/api_v2/tidb.log /tmp/tidb_cdc_test/api_v2/tikv_down.log /tmp/tidb_cdc_test/bank/pd1.log /tmp/tidb_cdc_test/bank/tikv_down/db/000028.log /tmp/tidb_cdc_test/bank/tidb_other.log /tmp/tidb_cdc_test/bank/pd1/region-meta/000005.log /tmp/tidb_cdc_test/bank/pd1/hot-region/000004.log /tmp/tidb_cdc_test/bank/tikv1/db/000018.log /tmp/tidb_cdc_test/bank/tikv1/db/000025.log /tmp/tidb_cdc_test/bank/tiflash/log/server.log /tmp/tidb_cdc_test/bank/tiflash/log/error.log /tmp/tidb_cdc_test/bank/tiflash/log/proxy.log /tmp/tidb_cdc_test/bank/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/bank/down_pd/region-meta/000005.log /tmp/tidb_cdc_test/bank/down_pd/hot-region/000004.log /tmp/tidb_cdc_test/bank/tikv3.log /tmp/tidb_cdc_test/bank/cdc.log table sorter.check4 not exists for 2-th check, retry later /tmp/tidb_cdc_test/bank/tidb_down.log /tmp/tidb_cdc_test/bank/tikv1.log /tmp/tidb_cdc_test/bank/tikv3/db/000018.log /tmp/tidb_cdc_test/bank/tikv3/db/000025.log /tmp/tidb_cdc_test/bank/tikv2/db/000032.log /tmp/tidb_cdc_test/bank/tikv2/db/000028.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e450001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30696, start at 2024-05-31 17:05:19.16853234 +0800 CST m=+4.300127352 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... /tmp/tidb_cdc_test/bank/tidb-slow.log /tmp/tidb_cdc_test/bank/tikv2.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795fa740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24255, start at 2024-05-31 17:05:20.579151566 +0800 CST m=+4.413808404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table sorter.check4 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:05:46 CST 2024] <<<<<< run test case sorter success! >>>>>> /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/bank/tikv2-2024-05-31T17-05-10.419.log [2024/05/31 17:05:45.093 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column name not equal, one is ag2o9 another is c"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... /tmp/tidb_cdc_test/bank/down_pd.log /tmp/tidb_cdc_test/bank/stdout.log /tmp/tidb_cdc_test/bank/tidb.log /tmp/tidb_cdc_test/bank/tikv_down.log start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + ls -alh log-G0.tar.gz -rw-r--r--. 1 jenkins jenkins 54M May 31 17:05 log-G0.tar.gz [Pipeline] archiveArtifacts Archiving artifacts + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later table multi_tables_ddl_test.t55 not exists for 6-th check, retry later table new_ci_collation_with_old_value_test.t1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table new_ci_collation_with_old_value_test.t1 exists table new_ci_collation_with_old_value_test.t2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table multi_tables_ddl_test.t55 not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Recording fingerprints [Pipeline] } [2024/05/31 17:05:55.781 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=201781432] ["get source checksum cost"=1.497999ms] ["get target checksum cost"=3.050528ms] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout table new_ci_collation_with_old_value_test.t2 exists table new_ci_collation_with_old_value_test.t3 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G0' table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... table new_ci_collation_with_old_value_test.t3 exists table new_ci_collation_with_old_value_test.t4 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_with_old_value_test.t4 exists table new_ci_collation_with_old_value_test.t5 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795f82c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24174, start at 2024-05-31 17:05:20.440301554 +0800 CST m=+4.339586980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795fa740019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:24255, start at 2024-05-31 17:05:20.579151566 +0800 CST m=+4.413808404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:20.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:20.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:20.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:06:02 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2817828180.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379872d0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37647, start at 2024-05-31 17:06:01.023170114 +0800 CST m=+4.198426000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:01.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:01.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:01.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e6c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30625, start at 2024-05-31 17:05:19.302839656 +0800 CST m=+4.490371689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3795e450001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:30696, start at 2024-05-31 17:05:19.16853234 +0800 CST m=+4.300127352 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:07:19.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:05:19.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:55:19.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.34703.out cli tso query --pd=http://127.0.0.1:2379 table new_ci_collation_with_old_value_test.t5 exists check diff failed 1-th time, retry later table multi_tables_ddl_test.finish_mark not exists for 4-th check, retry later + set +x + tso='450139616388579329 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139616388579329 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting CDC state checker... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:06:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63bebd8e-8b5a-4f88-a501-34dc72a957af {"id":"63bebd8e-8b5a-4f88-a501-34dc72a957af","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5ffa48e 63bebd8e-8b5a-4f88-a501-34dc72a957af /tidb/cdc/default/default/upstream/7375087255994384618 {"id":7375087255994384618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63bebd8e-8b5a-4f88-a501-34dc72a957af {"id":"63bebd8e-8b5a-4f88-a501-34dc72a957af","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5ffa48e 63bebd8e-8b5a-4f88-a501-34dc72a957af /tidb/cdc/default/default/upstream/7375087255994384618 {"id":7375087255994384618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63bebd8e-8b5a-4f88-a501-34dc72a957af {"id":"63bebd8e-8b5a-4f88-a501-34dc72a957af","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5ffa48e 63bebd8e-8b5a-4f88-a501-34dc72a957af /tidb/cdc/default/default/upstream/7375087255994384618 {"id":7375087255994384618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.28233.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: ed68ac3e-fe2e-4180-9ba0-7242b2cd08cf Info: {"upstream_id":7375087255994384618,"namespace":"default","id":"ed68ac3e-fe2e-4180-9ba0-7242b2cd08cf","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:06:06.258495521+08:00","start_ts":450139617003569157,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139617003569157,"checkpoint_ts":450139617003569157,"checkpoint_time":"2024-05-31 17:06:06.133"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table multi_tables_ddl_test.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798cb140003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26927, start at 2024-05-31 17:06:06.667122346 +0800 CST m=+5.570302116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:06.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:06.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:06.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:06:08 CST 2024] <<<<<< run test case new_ci_collation_with_old_value success! >>>>>> table multi_tables_ddl_test.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d9f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48773, start at 2024-05-31 17:06:07.627506718 +0800 CST m=+4.331938206 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.612 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.612 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log table multi_tables_ddl_test.finish_mark not exists for 7-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting CDC state checker... table multi_tables_ddl_test.finish_mark not exists for 8-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table multi_tables_ddl_test.finish_mark not exists for 9-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 10-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_without_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null }' + echo '{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null }' { "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null } ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450139620118364162, '"checkpoint_time":' '"2024-05-31' '17:06:18.015",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450139620118364162, '"checkpoint_time":' '"2024-05-31' '17:06:18.015",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null }' + echo '{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null }' { "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450139620118364162, "checkpoint_time": "2024-05-31 17:06:18.015", "error": null } ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450139620118364162, '"checkpoint_time":' '"2024-05-31' '17:06:18.015",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450139620118364162, '"checkpoint_time":' '"2024-05-31' '17:06:18.015",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139610681180170, "checkpoint_time": "2024-05-31 17:05:42.015", "error": { "time": "2024-05-31T17:05:44.111675481+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139611179253770]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139610681180170, "checkpoint_time": "2024-05-31 17:05:42.015", "error": { "time": "2024-05-31T17:05:44.111675481+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139611179253770]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7375086902124195218, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450139610681180170, "checkpoint_time": "2024-05-31 17:05:42.015", "error": { "time": "2024-05-31T17:05:44.111675481+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139611179253770]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450139610681180170, '"checkpoint_time":' '"2024-05-31' '17:05:42.015",' '"error":' '{' '"time":' '"2024-05-31T17:05:44.111675481+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450139611179253770]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375086902124195218, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450139610681180170, '"checkpoint_time":' '"2024-05-31' '17:05:42.015",' '"error":' '{' '"time":' '"2024-05-31T17:05:44.111675481+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[450139611179253770]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139611179253770]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-05-31 17:05:43.915 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 450139611179253770. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[450139611179253770]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:06:19 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table ddl_only_block_related_table.finish_mark exists /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (35781) - No such process wait process 35781 exit for 1-th time... process 35781 already exit [Fri May 31 17:06:22 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3599836000.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:06:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc000f5d830 TaskPositions:map[24012cd8-25c2-484d-82c6-0a483677d614:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 84, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 86, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 88, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24012cd8-25c2-484d-82c6-0a483677d614 {"id":"24012cd8-25c2-484d-82c6-0a483677d614","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dad9 24012cd8-25c2-484d-82c6-0a483677d614 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622025199621,"min-table-barrier-ts":450139622025199621,"admin-job-type":0} /tidb/cdc/default/default/task/position/24012cd8-25c2-484d-82c6-0a483677d614/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc000f5d830 TaskPositions:map[24012cd8-25c2-484d-82c6-0a483677d614:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 84, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 86, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 88, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24012cd8-25c2-484d-82c6-0a483677d614 {"id":"24012cd8-25c2-484d-82c6-0a483677d614","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dad9 24012cd8-25c2-484d-82c6-0a483677d614 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622025199621,"min-table-barrier-ts":450139622025199621,"admin-job-type":0} /tidb/cdc/default/default/task/position/24012cd8-25c2-484d-82c6-0a483677d614/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc000f5d830 TaskPositions:map[24012cd8-25c2-484d-82c6-0a483677d614:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 84, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 86, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating tableID: 88, resolvedTs: 450139622038044674, checkpointTs: 450139622025199621, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24012cd8-25c2-484d-82c6-0a483677d614 {"id":"24012cd8-25c2-484d-82c6-0a483677d614","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dad9 24012cd8-25c2-484d-82c6-0a483677d614 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622025199621,"min-table-barrier-ts":450139622025199621,"admin-job-type":0} /tidb/cdc/default/default/task/position/24012cd8-25c2-484d-82c6-0a483677d614/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later [Fri May 31 17:06:19 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.3316333165.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:06:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b136522a-af6c-450e-a051-8c47df8d6bc1 {"id":"b136522a-af6c-450e-a051-8c47df8d6bc1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5557c73 b136522a-af6c-450e-a051-8c47df8d6bc1 /tidb/cdc/default/default/upstream/7375087066994591564 {"id":7375087066994591564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b136522a-af6c-450e-a051-8c47df8d6bc1 {"id":"b136522a-af6c-450e-a051-8c47df8d6bc1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5557c73 b136522a-af6c-450e-a051-8c47df8d6bc1 /tidb/cdc/default/default/upstream/7375087066994591564 {"id":7375087066994591564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b136522a-af6c-450e-a051-8c47df8d6bc1 {"id":"b136522a-af6c-450e-a051-8c47df8d6bc1","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde5557c73 b136522a-af6c-450e-a051-8c47df8d6bc1 /tidb/cdc/default/default/upstream/7375087066994591564 {"id":7375087066994591564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3 -s + info='{ "upstream_id": 7375087066994591564, "namespace": "default", "id": "6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3", "state": "normal", "checkpoint_tso": 450139621325537284, "checkpoint_time": "2024-05-31 17:06:22.620", "error": null }' + echo '{ "upstream_id": 7375087066994591564, "namespace": "default", "id": "6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3", "state": "normal", "checkpoint_tso": 450139621325537284, "checkpoint_time": "2024-05-31 17:06:22.620", "error": null }' { "upstream_id": 7375087066994591564, "namespace": "default", "id": "6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3", "state": "normal", "checkpoint_tso": 450139621325537284, "checkpoint_time": "2024-05-31 17:06:22.620", "error": null } ++ echo '{' '"upstream_id":' 7375087066994591564, '"namespace":' '"default",' '"id":' '"6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3",' '"state":' '"normal",' '"checkpoint_tso":' 450139621325537284, '"checkpoint_time":' '"2024-05-31' '17:06:22.620",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375087066994591564, '"namespace":' '"default",' '"id":' '"6ed0cb83-0720-4f63-a0d8-fb5538cdf4d3",' '"state":' '"normal",' '"checkpoint_tso":' 450139621325537284, '"checkpoint_time":' '"2024-05-31' '17:06:22.620",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379872d0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37647, start at 2024-05-31 17:06:01.023170114 +0800 CST m=+4.198426000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:01.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:01.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:01.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:06:28 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table1 exists table test.table2 not exists for 1-th check, retry later check diff successfully check_ts_not_forward ddl-only-block-related-table *************************** 1. row *************************** count(distinct region_id): 1 table region_merge.t1 exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.table2 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3cf80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30503, start at 2024-05-31 17:06:30.358977676 +0800 CST m=+4.202770328 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d9f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48773, start at 2024-05-31 17:06:07.627506718 +0800 CST m=+4.331938206 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.612 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.612 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... table test.table2 exists check diff successfully wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:06:34 CST 2024] <<<<<< run test case region_merge success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiDB... check diff successfully Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798cb140003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26927, start at 2024-05-31 17:06:06.667122346 +0800 CST m=+5.570302116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:06.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:06.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:06.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Resume changefeed with checkpointTs3 450139625406595080 table test.table5 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table test.table5 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table test.table5 exists table test.table6 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0e880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48812, start at 2024-05-31 17:06:43.790188613 +0800 CST m=+4.249884281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.796 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/error.log chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table test.table6 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379871bc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37553, start at 2024-05-31 17:06:00.945105003 +0800 CST m=+4.210384026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:00.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:00.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:00.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379872d0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:37647, start at 2024-05-31 17:06:01.023170114 +0800 CST m=+4.198426000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:01.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:01.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:01.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:06:46 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.4171141713.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully The 1 times to try to start tidb cluster... Starting CDC state checker... Resume changefeed with checkpointTs1 450139608825987075 table test.table3 not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:06:49 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5c02f92-32cf-46b9-9a27-1b6f043b0b69 {"id":"d5c02f92-32cf-46b9-9a27-1b6f043b0b69","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c590 d5c02f92-32cf-46b9-9a27-1b6f043b0b69 /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5c02f92-32cf-46b9-9a27-1b6f043b0b69 {"id":"d5c02f92-32cf-46b9-9a27-1b6f043b0b69","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c590 d5c02f92-32cf-46b9-9a27-1b6f043b0b69 /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5c02f92-32cf-46b9-9a27-1b6f043b0b69 {"id":"d5c02f92-32cf-46b9-9a27-1b6f043b0b69","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c590 d5c02f92-32cf-46b9-9a27-1b6f043b0b69 /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798da080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48683, start at 2024-05-31 17:06:07.62299719 +0800 CST m=+4.379829485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d9f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2ttk2, pid:48773, start at 2024-05-31 17:06:07.627506718 +0800 CST m=+4.331938206 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.612 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.612 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.52965.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x + tso='450139629007142913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139629007142913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b81f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34461, start at 2024-05-31 17:06:51.174228916 +0800 CST m=+4.402575284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.183 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.183 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log table test.table3 not exists for 3-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3cf80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30503, start at 2024-05-31 17:06:30.358977676 +0800 CST m=+4.202770328 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table test.table3 exists table test.table4 not exists for 1-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table4 exists check diff failed 1-th time, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:06:59 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.5284952851.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad05466-5bfb-447d-a9f1-1b2d1acb6aff {"id":"dad05466-5bfb-447d-a9f1-1b2d1acb6aff","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d876e dad05466-5bfb-447d-a9f1-1b2d1acb6aff /tidb/cdc/default/default/changefeed/info/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"upstream-id":7375087094730733952,"namespace":"default","changefeed-id":"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:05:34.772576351+08:00","start-ts":450139608825987075,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-31T17:07:00.076994501+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139631134441478} /tidb/cdc/default/default/changefeed/status/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"checkpoint-ts":450139625419440164,"min-table-barrier-ts":450139625589833752,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad05466-5bfb-447d-a9f1-1b2d1acb6aff {"id":"dad05466-5bfb-447d-a9f1-1b2d1acb6aff","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d876e dad05466-5bfb-447d-a9f1-1b2d1acb6aff /tidb/cdc/default/default/changefeed/info/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"upstream-id":7375087094730733952,"namespace":"default","changefeed-id":"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:05:34.772576351+08:00","start-ts":450139608825987075,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-31T17:07:00.076994501+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139631134441478} /tidb/cdc/default/default/changefeed/status/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"checkpoint-ts":450139625419440164,"min-table-barrier-ts":450139625589833752,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dad05466-5bfb-447d-a9f1-1b2d1acb6aff {"id":"dad05466-5bfb-447d-a9f1-1b2d1acb6aff","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcde57d876e dad05466-5bfb-447d-a9f1-1b2d1acb6aff /tidb/cdc/default/default/changefeed/info/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"upstream-id":7375087094730733952,"namespace":"default","changefeed-id":"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:05:34.772576351+08:00","start-ts":450139608825987075,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-05-31T17:07:00.076994501+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139631134441478} /tidb/cdc/default/default/changefeed/status/ac5d38e9-b0e9-4cba-bcd5-76b8913e850b {"checkpoint-ts":450139625419440164,"min-table-barrier-ts":450139625589833752,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375087094730733952 {"id":7375087094730733952,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 ac5d38e9-b0e9-4cba-bcd5-76b8913e850b failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=ac5d38e9-b0e9-4cba-bcd5-76b8913e850b + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ac5d38e9-b0e9-4cba-bcd5-76b8913e850b -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "failed", "checkpoint_tso": 450139625419440164, "checkpoint_time": "2024-05-31 17:06:38.237", "error": { "time": "2024-05-31T17:07:00.076994501+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "failed", "checkpoint_tso": 450139625419440164, "checkpoint_time": "2024-05-31 17:06:38.237", "error": { "time": "2024-05-31T17:07:00.076994501+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "failed", "checkpoint_tso": 450139625419440164, "checkpoint_time": "2024-05-31 17:06:38.237", "error": { "time": "2024-05-31T17:07:00.076994501+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375087094730733952, '"namespace":' '"default",' '"id":' '"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b",' '"state":' '"failed",' '"checkpoint_tso":' 450139625419440164, '"checkpoint_time":' '"2024-05-31' '17:06:38.237",' '"error":' '{' '"time":' '"2024-05-31T17:07:00.076994501+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375087094730733952, '"namespace":' '"default",' '"id":' '"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b",' '"state":' '"failed",' '"checkpoint_tso":' 450139625419440164, '"checkpoint_time":' '"2024-05-31' '17:06:38.237",' '"error":' '{' '"time":' '"2024-05-31T17:07:00.076994501+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully check_changefeed_state http://127.0.0.1:2379 ac5d38e9-b0e9-4cba-bcd5-76b8913e850b normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ac5d38e9-b0e9-4cba-bcd5-76b8913e850b + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ac5d38e9-b0e9-4cba-bcd5-76b8913e850b -s table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later + info='{ "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "normal", "checkpoint_tso": 450139631986147332, "checkpoint_time": "2024-05-31 17:07:03.287", "error": null }' + echo '{ "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "normal", "checkpoint_tso": 450139631986147332, "checkpoint_time": "2024-05-31 17:07:03.287", "error": null }' { "upstream_id": 7375087094730733952, "namespace": "default", "id": "ac5d38e9-b0e9-4cba-bcd5-76b8913e850b", "state": "normal", "checkpoint_tso": 450139631986147332, "checkpoint_time": "2024-05-31 17:07:03.287", "error": null } ++ echo '{' '"upstream_id":' 7375087094730733952, '"namespace":' '"default",' '"id":' '"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b",' '"state":' '"normal",' '"checkpoint_tso":' 450139631986147332, '"checkpoint_time":' '"2024-05-31' '17:07:03.287",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375087094730733952, '"namespace":' '"default",' '"id":' '"ac5d38e9-b0e9-4cba-bcd5-76b8913e850b",' '"state":' '"normal",' '"checkpoint_tso":' 450139631986147332, '"checkpoint_time":' '"2024-05-31' '17:07:03.287",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table test1.table1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798d3e80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26906, start at 2024-05-31 17:06:07.267875526 +0800 CST m=+6.329443017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:07.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:07.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:07.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3798cb140003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:26927, start at 2024-05-31 17:06:06.667122346 +0800 CST m=+5.570302116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:06.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:06.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:06.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30455.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d78000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39667, start at 2024-05-31 17:07:04.190943393 +0800 CST m=+5.776434472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log + set +x + tso='450139632852271105 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139632852271105 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139632852271105 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:07:07 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3047530477.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table event_filter.t_truncate exists table test1.table1 not exists for 2-th check, retry later table event_filter.t_alter not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0e880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48812, start at 2024-05-31 17:06:43.790188613 +0800 CST m=+4.249884281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.796 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test1.table1 exists table event_filter.t_alter exists Starting CDC state checker... table test1.table2 not exists for 1-th check, retry later Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60940836-44de-4e64-bda3-dccb636519d8 {"id":"60940836-44de-4e64-bda3-dccb636519d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6f6818a 60940836-44de-4e64-bda3-dccb636519d8 /tidb/cdc/default/default/upstream/7375087448483950399 {"id":7375087448483950399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60940836-44de-4e64-bda3-dccb636519d8 {"id":"60940836-44de-4e64-bda3-dccb636519d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6f6818a 60940836-44de-4e64-bda3-dccb636519d8 /tidb/cdc/default/default/upstream/7375087448483950399 {"id":7375087448483950399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60940836-44de-4e64-bda3-dccb636519d8 {"id":"60940836-44de-4e64-bda3-dccb636519d8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6f6818a 60940836-44de-4e64-bda3-dccb636519d8 /tidb/cdc/default/default/upstream/7375087448483950399 {"id":7375087448483950399,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139632852271105 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30521.out cli changefeed create --start-ts=450139632852271105 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375087448483950399,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:07:10.63400313+08:00","start_ts":450139632852271105,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139632852271105,"checkpoint_ts":450139632852271105,"checkpoint_time":"2024-05-31 17:07:06.591"} PASS table event_filter.finish_mark exists check diff failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' table test1.table2 exists [Fri May 31 17:07:02 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.3482934831.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:05 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/4be51eaf-a398-4c7a-a95d-1fda9814b982 {"id":"4be51eaf-a398-4c7a-a95d-1fda9814b982","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fcde5feaf19 4be51eaf-a398-4c7a-a95d-1fda9814b982 /tidb/cdc/test1/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/4be51eaf-a398-4c7a-a95d-1fda9814b982 {"id":"4be51eaf-a398-4c7a-a95d-1fda9814b982","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fcde5feaf19 4be51eaf-a398-4c7a-a95d-1fda9814b982 /tidb/cdc/test1/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/4be51eaf-a398-4c7a-a95d-1fda9814b982 {"id":"4be51eaf-a398-4c7a-a95d-1fda9814b982","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318fcde5feaf19 4be51eaf-a398-4c7a-a95d-1fda9814b982 /tidb/cdc/test1/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:07:05 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.3489134893.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:08 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/d00c06dd-a19d-4efd-808f-15293ba548e6 {"id":"d00c06dd-a19d-4efd-808f-15293ba548e6","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fcde5feaf2f d00c06dd-a19d-4efd-808f-15293ba548e6 /tidb/cdc/test2/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/d00c06dd-a19d-4efd-808f-15293ba548e6 {"id":"d00c06dd-a19d-4efd-808f-15293ba548e6","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fcde5feaf2f d00c06dd-a19d-4efd-808f-15293ba548e6 /tidb/cdc/test2/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/d00c06dd-a19d-4efd-808f-15293ba548e6 {"id":"d00c06dd-a19d-4efd-808f-15293ba548e6","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318fcde5feaf2f d00c06dd-a19d-4efd-808f-15293ba548e6 /tidb/cdc/test2/default/upstream/7375087251983668506 {"id":7375087251983668506,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.34954.out cli changefeed create --start-ts=450139616388579329 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: 69a4df3f-9c79-44ca-b472-f80cc5496221 Info: {"upstream_id":7375087251983668506,"namespace":"default","id":"69a4df3f-9c79-44ca-b472-f80cc5496221","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:07:09.208521447+08:00","start_ts":450139616388579329,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139616388579329,"checkpoint_ts":450139616388579329,"checkpoint_time":"2024-05-31 17:06:03.787"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.35005.out cli changefeed create --start-ts=450139616388579329 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: 2bf851ce-113b-4958-9f5a-d1871b601b5b Info: {"upstream_id":7375087251983668506,"namespace":"default","id":"2bf851ce-113b-4958-9f5a-d1871b601b5b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:07:09.822141906+08:00","start_ts":450139616388579329,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139616388579329,"checkpoint_ts":450139616388579329,"checkpoint_time":"2024-05-31 17:06:03.787"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.multi_cdc1 not exists for 1-th check, retry later check diff successfully Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3d180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30431, start at 2024-05-31 17:06:30.343628175 +0800 CST m=+4.265994639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.350 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379a3cf80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--391r0, pid:30503, start at 2024-05-31 17:06:30.358977676 +0800 CST m=+4.202770328 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:30.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:30.334 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:30.334 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:07:12 CST 2024] <<<<<< START cdc server in new_ci_collation_without_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_without_old_value.3439134393.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:07:13 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:07:15 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:07:15 CST 2024] <<<<<< run test case event_filter success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:15 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f61a609-0b93-48ae-a52e-275de74bbbea {"id":"0f61a609-0b93-48ae-a52e-275de74bbbea","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7113c88 0f61a609-0b93-48ae-a52e-275de74bbbea /tidb/cdc/default/default/upstream/7375087556224576703 {"id":7375087556224576703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f61a609-0b93-48ae-a52e-275de74bbbea {"id":"0f61a609-0b93-48ae-a52e-275de74bbbea","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7113c88 0f61a609-0b93-48ae-a52e-275de74bbbea /tidb/cdc/default/default/upstream/7375087556224576703 {"id":7375087556224576703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f61a609-0b93-48ae-a52e-275de74bbbea {"id":"0f61a609-0b93-48ae-a52e-275de74bbbea","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7113c88 0f61a609-0b93-48ae-a52e-275de74bbbea /tidb/cdc/default/default/upstream/7375087556224576703 {"id":7375087556224576703,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ed7ae0e0-8b92-4195-9714-09662e73d568 Info: {"upstream_id":7375087556224576703,"namespace":"default","id":"ed7ae0e0-8b92-4195-9714-09662e73d568","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-31T17:07:15.63698477+08:00","start_ts":450139634380570625,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139634380570625,"checkpoint_ts":450139634380570625,"checkpoint_time":"2024-05-31 17:07:12.421"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b81f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34461, start at 2024-05-31 17:06:51.174228916 +0800 CST m=+4.402575284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.183 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.183 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_changefeed/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0f500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48719, start at 2024-05-31 17:06:43.821039436 +0800 CST m=+4.347458647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b0e880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--48fns, pid:48812, start at 2024-05-31 17:06:43.790188613 +0800 CST m=+4.249884281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:43.796 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:43.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:43.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:07:26 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5295552957.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:07:29 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5302853030.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:32 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:07:32 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5309553097.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:07:30.277 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.496072353s] [2024/05/31 17:07:30.279 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:35 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4447f7db-fdcb-4cee-82c7-53b99d7c4f31 {"id":"4447f7db-fdcb-4cee-82c7-53b99d7c4f31","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455ec3 4447f7db-fdcb-4cee-82c7-53b99d7c4f31 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4447f7db-fdcb-4cee-82c7-53b99d7c4f31 {"id":"4447f7db-fdcb-4cee-82c7-53b99d7c4f31","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455ec3 4447f7db-fdcb-4cee-82c7-53b99d7c4f31 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/326da764-5da2-4a71-aa5c-127524915cdb {"id":"326da764-5da2-4a71-aa5c-127524915cdb","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/4447f7db-fdcb-4cee-82c7-53b99d7c4f31 {"id":"4447f7db-fdcb-4cee-82c7-53b99d7c4f31","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/6a59dd88-3f9f-4fc1-bc15-592dc8794191 {"id":"6a59dd88-3f9f-4fc1-bc15-592dc8794191","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455e8e 6a59dd88-3f9f-4fc1-bc15-592dc8794191 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455eb1 326da764-5da2-4a71-aa5c-127524915cdb /tidb/cdc/default/__cdc_meta__/owner/22318fcde7455ec3 4447f7db-fdcb-4cee-82c7-53b99d7c4f31 /tidb/cdc/default/default/upstream/7375087606201295174 {"id":7375087606201295174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.cli.53157.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b811c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34371, start at 2024-05-31 17:06:51.112274044 +0800 CST m=+4.392319963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.119 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379b81f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:34461, start at 2024-05-31 17:06:51.174228916 +0800 CST m=+4.402575284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:08:51.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:06:51.183 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:56:51.183 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Create changefeed successfully! ID: 27cde9f3-04a5-4547-89e3-bbb93abdb2cf Info: {"upstream_id":7375087606201295174,"namespace":"default","id":"27cde9f3-04a5-4547-89e3-bbb93abdb2cf","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-31T17:07:36.211189675+08:00","start_ts":450139640592859149,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139640592859149,"checkpoint_ts":450139640592859149,"checkpoint_time":"2024-05-31 17:07:36.119"} PASS [Fri May 31 17:07:36 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.3838338385.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:07:36.306 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:07:36 CST 2024] <<<<<< run test case cdc success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29513, start at 2024-05-31 17:07:37.189146966 +0800 CST m=+4.444117661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.195 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e5130000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:36079, start at 2024-05-31 17:07:37.179913485 +0800 CST m=+4.520402676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.164 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.164 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5b6a16-b0e2-4fc5-855f-10d30d927e4f {"id":"0d5b6a16-b0e2-4fc5-855f-10d30d927e4f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7660590 0d5b6a16-b0e2-4fc5-855f-10d30d927e4f /tidb/cdc/default/default/upstream/7375087642830977161 {"id":7375087642830977161,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5b6a16-b0e2-4fc5-855f-10d30d927e4f {"id":"0d5b6a16-b0e2-4fc5-855f-10d30d927e4f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7660590 0d5b6a16-b0e2-4fc5-855f-10d30d927e4f /tidb/cdc/default/default/upstream/7375087642830977161 {"id":7375087642830977161,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5b6a16-b0e2-4fc5-855f-10d30d927e4f {"id":"0d5b6a16-b0e2-4fc5-855f-10d30d927e4f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7660590 0d5b6a16-b0e2-4fc5-855f-10d30d927e4f /tidb/cdc/default/default/upstream/7375087642830977161 {"id":7375087642830977161,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d78000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39667, start at 2024-05-31 17:07:04.190943393 +0800 CST m=+5.776434472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea300000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54112, start at 2024-05-31 17:07:42.421848316 +0800 CST m=+7.877956484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log ***************** properties ***************** "operationcount"="0" "mysql.db"="consistent_replicate_ddl" "insertproportion"="0" "scanproportion"="0" "threadcount"="10" "readallfields"="true" "dotransactions"="false" "readproportion"="0" "fieldcount"="3" "mysql.user"="root" "recordcount"="50" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.602251ms INSERT - Takes(s): 0.0, Count: 50, OPS: 9406.8, Avg(us): 1972, Min(us): 1041, Max(us): 5351, 95th(us): 6000, 99th(us): 6000 table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting CDC state checker... table consistent_replicate_ddl.usertable1 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e5130000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:36079, start at 2024-05-31 17:07:37.179913485 +0800 CST m=+4.520402676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.164 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.164 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri May 31 17:07:50 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.5307453076.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 {"id":"8f15bcdb-f5d5-4b8f-bd64-0e25f411b773","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6bc4521 8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 /tidb/cdc/default/default/upstream/7375087450840285283 {"id":7375087450840285283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 {"id":"8f15bcdb-f5d5-4b8f-bd64-0e25f411b773","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6bc4521 8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 /tidb/cdc/default/default/upstream/7375087450840285283 {"id":7375087450840285283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 {"id":"8f15bcdb-f5d5-4b8f-bd64-0e25f411b773","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6bc4521 8f15bcdb-f5d5-4b8f-bd64-0e25f411b773 /tidb/cdc/default/default/upstream/7375087450840285283 {"id":7375087450840285283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.53135.out cli changefeed create --start-ts=450139629007142913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a74500a9-4f40-4d5d-8976-147badf5fb46 Info: {"upstream_id":7375087450840285283,"namespace":"default","id":"a74500a9-4f40-4d5d-8976-147badf5fb46","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:07:53.666620767+08:00","start_ts":450139629007142913,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139629007142913,"checkpoint_ts":450139629007142913,"checkpoint_time":"2024-05-31 17:06:51.923"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table split_region.test1 not exists for 1-th check, retry later table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table split_region.test2 exists check diff successfully table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa7d8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30233, start at 2024-05-31 17:07:59.107575358 +0800 CST m=+4.239750234 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.114 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable_bak exists Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d9c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39617, start at 2024-05-31 17:07:04.199989519 +0800 CST m=+5.871580420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379c4d78000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:39667, start at 2024-05-31 17:07:04.190943393 +0800 CST m=+5.776434472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:04.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:04.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:04.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.43427.out cli tso query --pd=http://127.0.0.1:2379 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + set +x + tso='450139647849267201 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139647849267201 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri May 31 17:08:04 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.4344343445.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 pass check, checkpoint tso not forward after 10s run task successfully wait process 36003 exit for 1-th time... wait process 36003 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (36003) - No such process wait process 36003 exit for 3-th time... process 36003 already exit [Fri May 31 17:07:55 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.3687336875.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:07:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc00301d2d8 TaskPositions:map[4c0f26a2-6fb9-4708-b6da-ea8efefc0d83:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 84, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 86, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 88, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 {"id":"4c0f26a2-6fb9-4708-b6da-ea8efefc0d83","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dbca 4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622287081482,"min-table-barrier-ts":450139646181507077,"admin-job-type":0} /tidb/cdc/default/default/task/position/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc00301d2d8 TaskPositions:map[4c0f26a2-6fb9-4708-b6da-ea8efefc0d83:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 84, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 86, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 88, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 {"id":"4c0f26a2-6fb9-4708-b6da-ea8efefc0d83","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dbca 4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} + grep -q 'failed to get info:' /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622287081482,"min-table-barrier-ts":450139646181507077,"admin-job-type":0} /tidb/cdc/default/default/task/position/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} Status:0xc00301d2d8 TaskPositions:map[4c0f26a2-6fb9-4708-b6da-ea8efefc0d83:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 84, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 86, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing tableID: 88, resolvedTs: 450139622287081482, checkpointTs: 450139622287081482, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 {"id":"4c0f26a2-6fb9-4708-b6da-ea8efefc0d83","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde548dbca 4c0f26a2-6fb9-4708-b6da-ea8efefc0d83 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375087052477624293,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-31T17:05:19.453294718+08:00","start-ts":450139604736802820,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139604763017219} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450139622287081482,"min-table-barrier-ts":450139646181507077,"admin-job-type":0} /tidb/cdc/default/default/task/position/4c0f26a2-6fb9-4708-b6da-ea8efefc0d83/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087052477624293 {"id":7375087052477624293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 450139622287081500->450139646706057200 checkpoint: 450139622287081500->450139646692687900 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:08:01 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:08:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79cde9f5-edda-430d-845a-7ad539e33f84 {"id":"79cde9f5-edda-430d-845a-7ad539e33f84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7d0bf91 79cde9f5-edda-430d-845a-7ad539e33f84 /tidb/cdc/default/default/upstream/7375087685193876449 {"id":7375087685193876449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79cde9f5-edda-430d-845a-7ad539e33f84 {"id":"79cde9f5-edda-430d-845a-7ad539e33f84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7d0bf91 79cde9f5-edda-430d-845a-7ad539e33f84 /tidb/cdc/default/default/upstream/7375087685193876449 {"id":7375087685193876449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79cde9f5-edda-430d-845a-7ad539e33f84 {"id":"79cde9f5-edda-430d-845a-7ad539e33f84","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde7d0bf91 79cde9f5-edda-430d-845a-7ad539e33f84 /tidb/cdc/default/default/upstream/7375087685193876449 {"id":7375087685193876449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.43500.out cli changefeed create --start-ts=450139647849267201 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic Create changefeed successfully! ID: 5b783a8f-15d6-405d-9b4a-5c70002d1d12 Info: {"upstream_id":7375087685193876449,"namespace":"default","id":"5b783a8f-15d6-405d-9b4a-5c70002d1d12","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-31T17:08:08.037540821+08:00","start_ts":450139647849267201,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139647849267201,"checkpoint_ts":450139647849267201,"checkpoint_time":"2024-05-31 17:08:03.800"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading k8s.io/api v0.24.0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:08:08 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.4201842020.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea300000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54112, start at 2024-05-31 17:07:42.421848316 +0800 CST m=+7.877956484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading k8s.io/apimachinery v0.24.0 start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... go: downloading k8s.io/klog/v2 v2.60.1 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading github.com/go-logr/logr v1.2.0 + kill_tikv ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ ps aux + info='jenkins 29638 10.1 0.3 3156800 1456980 ? Sl 17:07 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 29639 9.5 0.3 3151164 1446108 ? Sl 17:07 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 29640 9.5 0.3 3146628 1443056 ? Sl 17:07 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 29642 9.5 0.3 3148092 1445764 ? Sl 17:07 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ awk '{print $2}' ++ grep tikv-server ++ xargs kill -9 ++ grep /tmp/tidb_cdc_test/synced_status ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6080 0 --:--:-- --:--:-- --:--:-- 6230 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:08:08.491","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:08:04.641","now_ts":"2024-05-31 17:08:09.000","info":"The data syncing is not finished, please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:08:08.491","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:08:04.641","now_ts":"2024-05-31' '17:08:09.000","info":"The' data syncing is not finished, please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:08:08.491","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:08:04.641","now_ts":"2024-05-31' '17:08:09.000","info":"The' data syncing is not finished, please 'wait"}' + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29513, start at 2024-05-31 17:07:37.189146966 +0800 CST m=+4.444117661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.195 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:08:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9c76df61-c2d2-48cc-be83-1c818ea921d7 {ClusterID:default ID:default/9c76df61-c2d2-48cc-be83-1c818ea921d7 Info:{"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} Status:0xc000f7e720 TaskPositions:map[8299becc-5236-42f6-a722-aeddf095187a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 84, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 86, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 90, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 88, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8299becc-5236-42f6-a722-aeddf095187a {"id":"8299becc-5236-42f6-a722-aeddf095187a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c721 8299becc-5236-42f6-a722-aeddf095187a /tidb/cdc/default/default/changefeed/info/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} /tidb/cdc/default/default/changefeed/status/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":450139649520173063,"min-table-barrier-ts":450139649520173063,"admin-job-type":0} /tidb/cdc/default/default/task/position/8299becc-5236-42f6-a722-aeddf095187a/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9c76df61-c2d2-48cc-be83-1c818ea921d7 {ClusterID:default ID:default/9c76df61-c2d2-48cc-be83-1c818ea921d7 Info:{"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} Status:0xc000f7e720 TaskPositions:map[8299becc-5236-42f6-a722-aeddf095187a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 84, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 86, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 90, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 88, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8299becc-5236-42f6-a722-aeddf095187a {"id":"8299becc-5236-42f6-a722-aeddf095187a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c721 8299becc-5236-42f6-a722-aeddf095187a /tidb/cdc/default/default/changefeed/info/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} /tidb/cdc/default/default/changefeed/status/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":450139649520173063,"min-table-barrier-ts":450139649520173063,"admin-job-type":0} /tidb/cdc/default/default/task/position/8299becc-5236-42f6-a722-aeddf095187a/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9c76df61-c2d2-48cc-be83-1c818ea921d7 {ClusterID:default ID:default/9c76df61-c2d2-48cc-be83-1c818ea921d7 Info:{"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} Status:0xc000f7e720 TaskPositions:map[8299becc-5236-42f6-a722-aeddf095187a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 84, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 86, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 90, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating tableID: 88, resolvedTs: 450139649520173063, checkpointTs: 450139649520173063, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8299becc-5236-42f6-a722-aeddf095187a {"id":"8299becc-5236-42f6-a722-aeddf095187a","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde6a7c721 8299becc-5236-42f6-a722-aeddf095187a /tidb/cdc/default/default/changefeed/info/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"upstream-id":7375087430820061564,"namespace":"default","changefeed-id":"9c76df61-c2d2-48cc-be83-1c818ea921d7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:06:50.132967741+08:00","start-ts":450139628509331462,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139628535545859} /tidb/cdc/default/default/changefeed/status/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":450139649520173063,"min-table-barrier-ts":450139649520173063,"admin-job-type":0} /tidb/cdc/default/default/task/position/8299becc-5236-42f6-a722-aeddf095187a/9c76df61-c2d2-48cc-be83-1c818ea921d7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375087430820061564 {"id":7375087430820061564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table new_ci_collation_without_old_value_test.t1 not exists for 1-th check, retry later check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:08:12 CST 2024] <<<<<< run test case split_region success! >>>>>> table new_ci_collation_without_old_value_test.t1 exists table new_ci_collation_without_old_value_test.t2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table new_ci_collation_without_old_value_test.t2 exists table new_ci_collation_without_old_value_test.t3 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table new_ci_collation_without_old_value_test.t3 exists table new_ci_collation_without_old_value_test.t4 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:35998, start at 2024-05-31 17:07:37.157072286 +0800 CST m=+4.557196727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e5130000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d14t8, pid:36079, start at 2024-05-31 17:07:37.179913485 +0800 CST m=+4.520402676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.164 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.164 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:08:17 CST 2024] <<<<<< START cdc server in multi_changefeed case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_changefeed.4005940061.out server --log-file /tmp/tidb_cdc_test/multi_changefeed/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_changefeed/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_without_old_value_test.t4 exists table new_ci_collation_without_old_value_test.t5 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:08:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec55b993-63e6-43d7-ad00-d99c16e8a73b {"id":"ec55b993-63e6-43d7-ad00-d99c16e8a73b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde80ec48e ec55b993-63e6-43d7-ad00-d99c16e8a73b /tidb/cdc/default/default/upstream/7375087844362832758 {"id":7375087844362832758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec55b993-63e6-43d7-ad00-d99c16e8a73b {"id":"ec55b993-63e6-43d7-ad00-d99c16e8a73b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde80ec48e ec55b993-63e6-43d7-ad00-d99c16e8a73b /tidb/cdc/default/default/upstream/7375087844362832758 {"id":7375087844362832758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec55b993-63e6-43d7-ad00-d99c16e8a73b {"id":"ec55b993-63e6-43d7-ad00-d99c16e8a73b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde80ec48e ec55b993-63e6-43d7-ad00-d99c16e8a73b /tidb/cdc/default/default/upstream/7375087844362832758 {"id":7375087844362832758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: old-value-cf Info: {"upstream_id":7375087844362832758,"namespace":"default","id":"old-value-cf","sink_uri":"blackhole:","create_time":"2024-05-31T17:08:20.990548381+08:00","start_ts":450139652326948870,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139652326948870,"checkpoint_ts":450139652326948870,"checkpoint_time":"2024-05-31 17:08:20.881"} Create changefeed successfully! ID: no-old-value-cf Info: {"upstream_id":7375087844362832758,"namespace":"default","id":"no-old-value-cf","sink_uri":"blackhole:","create_time":"2024-05-31T17:08:21.189933657+08:00","start_ts":450139652379115522,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139652379115522,"checkpoint_ts":450139652379115522,"checkpoint_time":"2024-05-31 17:08:21.080"} table new_ci_collation_without_old_value_test.t5 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout check diff successfully [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa7d8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30233, start at 2024-05-31 17:07:59.107575358 +0800 CST m=+4.239750234 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.114 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // stage [Pipeline] } [Pipeline] // container check diff failed 1-th time, retry later [Pipeline] } [Pipeline] // withEnv ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [Pipeline] } [Pipeline] // node [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a1273c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38207, start at 2024-05-31 17:08:23.656716599 +0800 CST m=+4.158015047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:08:26 CST 2024] <<<<<< run test case new_ci_collation_without_old_value success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e510c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29439, start at 2024-05-31 17:07:37.159102631 +0800 CST m=+4.503849277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379e50f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:29513, start at 2024-05-31 17:07:37.189146966 +0800 CST m=+4.444117661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:37.195 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:37.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:37.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:08:29 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.3352233524.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea254000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54077, start at 2024-05-31 17:07:42.38312379 +0800 CST m=+7.930947290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379ea300000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:54112, start at 2024-05-31 17:07:42.421848316 +0800 CST m=+7.877956484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:42.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:42.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:42.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:08:29 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.5791157913.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 + workdir=/tmp/tidb_cdc_test/csv_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_partition_table ++ date + echo '[Fri May 31 17:08:31 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>>' [Fri May 31 17:08:31 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:08:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86cd96f8-9834-467d-8c98-ab4e95e1d11d {"id":"86cd96f8-9834-467d-8c98-ab4e95e1d11d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8389a8e 86cd96f8-9834-467d-8c98-ab4e95e1d11d /tidb/cdc/default/default/upstream/7375087843056830965 {"id":7375087843056830965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86cd96f8-9834-467d-8c98-ab4e95e1d11d {"id":"86cd96f8-9834-467d-8c98-ab4e95e1d11d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8389a8e 86cd96f8-9834-467d-8c98-ab4e95e1d11d /tidb/cdc/default/default/upstream/7375087843056830965 {"id":7375087843056830965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86cd96f8-9834-467d-8c98-ab4e95e1d11d {"id":"86cd96f8-9834-467d-8c98-ab4e95e1d11d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8389a8e 86cd96f8-9834-467d-8c98-ab4e95e1d11d /tidb/cdc/default/default/upstream/7375087843056830965 {"id":7375087843056830965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ba4bffb6-5e98-4358-a9d8-13f7171f4d41 Info: {"upstream_id":7375087843056830965,"namespace":"default","id":"ba4bffb6-5e98-4358-a9d8-13f7171f4d41","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-31T17:08:32.78018879+08:00","start_ts":450139654583484417,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139654583484417,"checkpoint_ts":450139654583484417,"checkpoint_time":"2024-05-31 17:08:29.489"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:08:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 {"id":"fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde83c618e fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 /tidb/cdc/default/default/upstream/7375087852406077498 {"id":7375087852406077498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 {"id":"fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde83c618e fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 /tidb/cdc/default/default/upstream/7375087852406077498 {"id":7375087852406077498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 {"id":"fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde83c618e fa1f8ba1-22f9-4c47-96c9-cd4c0f9f8797 /tidb/cdc/default/default/upstream/7375087852406077498 {"id":7375087852406077498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 801f654a-91d7-47eb-96a4-dee5d532037f Info: {"upstream_id":7375087852406077498,"namespace":"default","id":"801f654a-91d7-47eb-96a4-dee5d532037f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:08:33.193391762+08:00","start_ts":450139654654525442,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139654654525442,"checkpoint_ts":450139654654525442,"checkpoint_time":"2024-05-31 17:08:29.760"} start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri May 31 17:08:35 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv table partition_table.t not exists for 1-th check, retry later [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa8b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30163, start at 2024-05-31 17:07:59.178545187 +0800 CST m=+4.364764731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f379fa7d8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:30233, start at 2024-05-31 17:07:59.107575358 +0800 CST m=+4.239750234 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:09:59.114 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:07:59.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:57:59.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.34134.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + set +x + tso='450139658068426754 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139658068426754 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a1273c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38207, start at 2024-05-31 17:08:23.656716599 +0800 CST m=+4.158015047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table partition_table.finish_mark not exists for 4-th check, retry later table partition_table.finish_mark not exists for 5-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later table partition_table.finish_mark not exists for 7-th check, retry later table partition_table.finish_mark not exists for 8-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table partition_table.finish_mark not exists for 9-th check, retry later table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 exists table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later table partition_table.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:09:04 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a128140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38110, start at 2024-05-31 17:08:23.690598694 +0800 CST m=+4.243634220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a1273c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:38207, start at 2024-05-31 17:08:23.656716599 +0800 CST m=+4.158015047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:10:23.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:08:23.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-16:58:23.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.42289.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:09:00.757 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/05/31 17:09:00.761 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/31 17:09:00.761 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7375087685193876449] [2024/05/31 17:09:00.761 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/05/31 17:09:00.761 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/05/31 17:09:00.763 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/31 17:09:00.764 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7375087685193876449] [2024/05/31 17:09:00.764 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/05/31 17:09:00.765 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/31 17:09:00.766 +08:00] [INFO] [region_cache.go:2485] ["change store resolve state"] [store=1] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] + set +x + tso='450139664168517633 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139664168517633 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later table processor_delay.t19 not exists for 2-th check, retry later [2024/05/31 17:09:10.793 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/31 17:09:10.793 +08:00] [INFO] [client.go:783] ["[pd] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/31 17:09:10.793 +08:00] [INFO] [client.go:719] ["[pd] exit tso dispatcher"] [dc-location=global] [2024/05/31 17:09:10.793 +08:00] [INFO] [base_client.go:143] ["[pd] exit member loop due to context canceled"] table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir table test.t2 not exists for 1-th check, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed can't found update row with old value run task failed 1-th time, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later table test.t2 not exists for 2-th check, retry later check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed can't found update row with old value run task failed 2-th time, retry later table processor_delay.t23 exists table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later table test.t2 not exists for 3-th check, retry later table test.t2 not exists for 4-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed [2024/05/31 17:09:18.917 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666980536330,\"commit-ts\":450139666980536331,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/05/31 17:09:18.918 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666980536342,\"commit-ts\":450139666980536343,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/05/31 17:09:18.918 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666993643531,\"commit-ts\":450139666993643532,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] [2024/05/31 17:09:18.980 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666980536330,\"commit-ts\":450139666980536331,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/05/31 17:09:18.981 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666980536342,\"commit-ts\":450139666980536343,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/05/31 17:09:18.981 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":450139666993643531,\"commit-ts\":450139666993643532,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:09:23 CST 2024] <<<<<< run test case multi_changefeed success! >>>>>> table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table test.t2 not exists for 5-th check, retry later table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later table test.t2 not exists for 6-th check, retry later table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later table test.t2 not exists for 7-th check, retry later table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table test.t2 not exists for 8-th check, retry later table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later table test.t2 not exists for 9-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.t2 exists check diff successfully table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:09:36 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 table test.finish_mark not exists for 3-th check, retry later run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 table test.finish_mark not exists for 4-th check, retry later run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later table test.finish_mark not exists for 5-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later table test.finish_mark exists check table exists success check diff successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:09:48 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:09:48 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later try a VALID cdc server command [Fri May 31 17:09:39 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.3422934231.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:09:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84c3716b-117b-45c7-90a6-4eacfc4d51f0 {"id":"84c3716b-117b-45c7-90a6-4eacfc4d51f0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde86db60a 84c3716b-117b-45c7-90a6-4eacfc4d51f0 /tidb/cdc/default/default/upstream/7375087932052649477 {"id":7375087932052649477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84c3716b-117b-45c7-90a6-4eacfc4d51f0 {"id":"84c3716b-117b-45c7-90a6-4eacfc4d51f0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde86db60a 84c3716b-117b-45c7-90a6-4eacfc4d51f0 /tidb/cdc/default/default/upstream/7375087932052649477 {"id":7375087932052649477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84c3716b-117b-45c7-90a6-4eacfc4d51f0 {"id":"84c3716b-117b-45c7-90a6-4eacfc4d51f0","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde86db60a 84c3716b-117b-45c7-90a6-4eacfc4d51f0 /tidb/cdc/default/default/upstream/7375087932052649477 {"id":7375087932052649477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later Verifying downstream PD is started... table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri May 31 17:10:02 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ true != \n\o ]] + GO_FAILPOINTS= + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.3430234304.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73aa40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46865, start at 2024-05-31 17:10:03.214933913 +0800 CST m=+6.287958189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7936c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37243, start at 2024-05-31 17:10:08.860779696 +0800 CST m=+4.381924712 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t50 exists Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 1-th time, retry later [Fri May 31 17:10:03 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.4240442406.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:10:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba253557-d0e6-48c6-9490-e95c6bc1426d {"id":"ba253557-d0e6-48c6-9490-e95c6bc1426d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8ce781a ba253557-d0e6-48c6-9490-e95c6bc1426d /tidb/cdc/default/default/upstream/7375088037464537247 {"id":7375088037464537247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba253557-d0e6-48c6-9490-e95c6bc1426d {"id":"ba253557-d0e6-48c6-9490-e95c6bc1426d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8ce781a ba253557-d0e6-48c6-9490-e95c6bc1426d /tidb/cdc/default/default/upstream/7375088037464537247 {"id":7375088037464537247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba253557-d0e6-48c6-9490-e95c6bc1426d {"id":"ba253557-d0e6-48c6-9490-e95c6bc1426d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcde8ce781a ba253557-d0e6-48c6-9490-e95c6bc1426d /tidb/cdc/default/default/upstream/7375088037464537247 {"id":7375088037464537247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.42466.out cli changefeed create --start-ts=450139664168517633 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: be6dcf52-04dc-4a04-a7fa-a64a87b6e030 Info: {"upstream_id":7375088037464537247,"namespace":"default","id":"be6dcf52-04dc-4a04-a7fa-a64a87b6e030","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:10:07.342781362+08:00","start_ts":450139664168517633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139664168517633,"checkpoint_ts":450139664168517633,"checkpoint_time":"2024-05-31 17:09:06.053"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:10:16 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f5f4001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:59078, start at 2024-05-31 17:10:15.21205172 +0800 CST m=+5.915018138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "450139681541062660" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/31 17:10:14.017 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 17:10:14.018 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 17:10:14.018 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139681541062660] [resolvedTs=450139681541062660] [2024/05/31 17:10:14.018 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=281.353µs] [2024/05/31 17:10:14.018 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=274.533µs] [2024/05/31 17:10:14.018 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:10:14.018 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:10:14.022 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:10:14.024 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 17:10:14.025 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 17:10:14.025 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 17:10:14.025 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 17:10:14.028 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:10:14.029 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 17:10:14.029 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=450139681541062660] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 17:10:14.029 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Fri May 31 17:10:14 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri May 31 17:10:22 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D487C1F8ED1C97 < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 09:10:25 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_panic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73aa40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46865, start at 2024-05-31 17:10:03.214933913 +0800 CST m=+6.287958189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 23121 0 --:--:-- --:--:-- --:--:-- 23322 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:08:09.492","puller_resolved_ts":"2024-05-31 17:08:09.492","last_synced_ts":"2024-05-31 17:08:04.641","now_ts":"2024-05-31 17:10:19.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:08:09.492","puller_resolved_ts":"2024-05-31' '17:08:09.492","last_synced_ts":"2024-05-31' '17:08:04.641","now_ts":"2024-05-31' '17:10:19.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:08:09.492","puller_resolved_ts":"2024-05-31' '17:08:09.492","last_synced_ts":"2024-05-31' '17:08:04.641","now_ts":"2024-05-31' '17:10:19.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f5f4001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:59078, start at 2024-05-31 17:10:15.21205172 +0800 CST m=+5.915018138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a977f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40543, start at 2024-05-31 17:10:39.896682066 +0800 CST m=+4.253743362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_panic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_panic/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a99a940019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43390, start at 2024-05-31 17:10:42.126198826 +0800 CST m=+4.188282697 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7936c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37243, start at 2024-05-31 17:10:08.860779696 +0800 CST m=+4.381924712 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9df880008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35289, start at 2024-05-31 17:10:46.505860289 +0800 CST m=+4.203661916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73ccc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46817, start at 2024-05-31 17:10:03.315703996 +0800 CST m=+6.520551238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a73aa40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:46865, start at 2024-05-31 17:10:03.214933913 +0800 CST m=+6.287958189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:03.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:03.227 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:03.227 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.50864.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139691294130177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139691294130177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:10:49 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.5090350905.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa0ad00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31459, start at 2024-05-31 17:10:49.278642609 +0800 CST m=+6.005588086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:10:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67e4dc0-0d27-459c-836f-808413e63643 {"id":"d67e4dc0-0d27-459c-836f-808413e63643","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea5df58a d67e4dc0-0d27-459c-836f-808413e63643 /tidb/cdc/default/default/upstream/7375088452766647463 {"id":7375088452766647463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67e4dc0-0d27-459c-836f-808413e63643 {"id":"d67e4dc0-0d27-459c-836f-808413e63643","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea5df58a d67e4dc0-0d27-459c-836f-808413e63643 /tidb/cdc/default/default/upstream/7375088452766647463 {"id":7375088452766647463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d67e4dc0-0d27-459c-836f-808413e63643 {"id":"d67e4dc0-0d27-459c-836f-808413e63643","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea5df58a d67e4dc0-0d27-459c-836f-808413e63643 /tidb/cdc/default/default/upstream/7375088452766647463 {"id":7375088452766647463,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.50961.out cli changefeed create --start-ts=450139691294130177 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: cb35d329-2e4a-4943-af75-b5a573eed7f9 Info: {"upstream_id":7375088452766647463,"namespace":"default","id":"cb35d329-2e4a-4943-af75-b5a573eed7f9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:10:53.332807376+08:00","start_ts":450139691294130177,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139691294130177,"checkpoint_ts":450139691294130177,"checkpoint_time":"2024-05-31 17:10:49.529"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting CDC state checker... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f414000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:58998, start at 2024-05-31 17:10:15.05608742 +0800 CST m=+5.845798775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7f5f4001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--mz5fh, pid:59078, start at 2024-05-31 17:10:15.21205172 +0800 CST m=+5.915018138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:15.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:15.215 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:15.215 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63168.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139693946503169 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139693946503169 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a977f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40543, start at 2024-05-31 17:10:39.896682066 +0800 CST m=+4.253743362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a792900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37167, start at 2024-05-31 17:10:08.831519255 +0800 CST m=+4.444011943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a7936c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:37243, start at 2024-05-31 17:10:08.860779696 +0800 CST m=+4.381924712 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:08.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:08.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:08.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.41241.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139695192735745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139695192735745 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:11:04 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.4128641288.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a99a940019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43390, start at 2024-05-31 17:10:42.126198826 +0800 CST m=+4.188282697 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b3d432-5c12-4839-9a69-dc6a289dcec2 {"id":"77b3d432-5c12-4839-9a69-dc6a289dcec2","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea92688f 77b3d432-5c12-4839-9a69-dc6a289dcec2 /tidb/cdc/default/default/upstream/7375088495172720943 {"id":7375088495172720943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b3d432-5c12-4839-9a69-dc6a289dcec2 {"id":"77b3d432-5c12-4839-9a69-dc6a289dcec2","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea92688f 77b3d432-5c12-4839-9a69-dc6a289dcec2 /tidb/cdc/default/default/upstream/7375088495172720943 {"id":7375088495172720943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b3d432-5c12-4839-9a69-dc6a289dcec2 {"id":"77b3d432-5c12-4839-9a69-dc6a289dcec2","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea92688f 77b3d432-5c12-4839-9a69-dc6a289dcec2 /tidb/cdc/default/default/upstream/7375088495172720943 {"id":7375088495172720943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.41354.out cli changefeed create --start-ts=450139695192735745 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 4caa1864-2db0-4a1c-90b0-0c6bb4a28916 Info: {"upstream_id":7375088495172720943,"namespace":"default","id":"4caa1864-2db0-4a1c-90b0-0c6bb4a28916","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:11:08.094953581+08:00","start_ts":450139695192735745,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139695192735745,"checkpoint_ts":450139695192735745,"checkpoint_time":"2024-05-31 17:11:04.401"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9df880008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35289, start at 2024-05-31 17:10:46.505860289 +0800 CST m=+4.203661916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9763c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40451, start at 2024-05-31 17:10:39.79970457 +0800 CST m=+4.233041506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a977f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--2fv5k, pid:40543, start at 2024-05-31 17:10:39.896682066 +0800 CST m=+4.253743362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:39.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:39.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:39.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.44565.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... + set +x + tso='450139699468566529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139699468566529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa0ad00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31459, start at 2024-05-31 17:10:49.278642609 +0800 CST m=+6.005588086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Fri May 31 17:11:21 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ true == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + true + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.4460844610.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:11:24 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.446134467444676.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a999e80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43299, start at 2024-05-31 17:10:42.056734 +0800 CST m=+4.197751753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a99a940019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:43390, start at 2024-05-31 17:10:42.126198826 +0800 CST m=+4.188282697 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:42.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:42.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:42.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:11:26 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4751547517.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:27 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00124594-6545-4ed2-98d6-d58c7c3ae7b5 {"id":"00124594-6545-4ed2-98d6-d58c7c3ae7b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada53b0 00124594-6545-4ed2-98d6-d58c7c3ae7b5 /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00124594-6545-4ed2-98d6-d58c7c3ae7b5 {"id":"00124594-6545-4ed2-98d6-d58c7c3ae7b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada53b0 00124594-6545-4ed2-98d6-d58c7c3ae7b5 /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00124594-6545-4ed2-98d6-d58c7c3ae7b5 {"id":"00124594-6545-4ed2-98d6-d58c7c3ae7b5","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/9ff0b084-74af-4c88-bc43-e8cdc7f9688b {"id":"9ff0b084-74af-4c88-bc43-e8cdc7f9688b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada538d 9ff0b084-74af-4c88-bc43-e8cdc7f9688b /tidb/cdc/default/__cdc_meta__/owner/22318fcdeada53b0 00124594-6545-4ed2-98d6-d58c7c3ae7b5 /tidb/cdc/default/default/upstream/7375088623369835034 {"id":7375088623369835034,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.44742.out cli changefeed create --start-ts=450139699468566529 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: b5b82191-1f3c-4d08-b8c6-09f93a0b0e76 Info: {"upstream_id":7375088623369835034,"namespace":"default","id":"b5b82191-1f3c-4d08-b8c6-09f93a0b0e76","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:11:27.464199221+08:00","start_ts":450139699468566529,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139699468566529,"checkpoint_ts":450139699468566529,"checkpoint_time":"2024-05-31 17:11:20.712"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745a0a1f-7cb4-44a4-b439-43dbb810c49b {"id":"745a0a1f-7cb4-44a4-b439-43dbb810c49b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2c8e 745a0a1f-7cb4-44a4-b439-43dbb810c49b /tidb/cdc/default/default/upstream/7375088634849628583 {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745a0a1f-7cb4-44a4-b439-43dbb810c49b {"id":"745a0a1f-7cb4-44a4-b439-43dbb810c49b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2c8e 745a0a1f-7cb4-44a4-b439-43dbb810c49b /tidb/cdc/default/default/upstream/7375088634849628583 {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745a0a1f-7cb4-44a4-b439-43dbb810c49b {"id":"745a0a1f-7cb4-44a4-b439-43dbb810c49b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2c8e 745a0a1f-7cb4-44a4-b439-43dbb810c49b /tidb/cdc/default/default/upstream/7375088634849628583 {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9dd58001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35204, start at 2024-05-31 17:10:46.394480514 +0800 CST m=+4.146677292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37a9df880008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:35289, start at 2024-05-31 17:10:46.505860289 +0800 CST m=+4.203661916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:46.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:46.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:46.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa1300000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31428, start at 2024-05-31 17:10:49.811453535 +0800 CST m=+6.699706481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37aa0ad00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:31459, start at 2024-05-31 17:10:49.278642609 +0800 CST m=+6.005588086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:12:49.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:10:49.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:00:49.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.35014.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139707308507137 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139707308507137 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139707308507137 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:11:51 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3503735039.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da70af4e-8c01-4259-bba8-c84154f9b8fd {"id":"da70af4e-8c01-4259-bba8-c84154f9b8fd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeb3bf792 da70af4e-8c01-4259-bba8-c84154f9b8fd /tidb/cdc/default/default/upstream/7375088653279066961 {"id":7375088653279066961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da70af4e-8c01-4259-bba8-c84154f9b8fd {"id":"da70af4e-8c01-4259-bba8-c84154f9b8fd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeb3bf792 da70af4e-8c01-4259-bba8-c84154f9b8fd /tidb/cdc/default/default/upstream/7375088653279066961 {"id":7375088653279066961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da70af4e-8c01-4259-bba8-c84154f9b8fd {"id":"da70af4e-8c01-4259-bba8-c84154f9b8fd","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeb3bf792 da70af4e-8c01-4259-bba8-c84154f9b8fd /tidb/cdc/default/default/upstream/7375088653279066961 {"id":7375088653279066961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139707308507137 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.35080.out cli changefeed create --start-ts=450139707308507137 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375088653279066961,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:11:54.667103765+08:00","start_ts":450139707308507137,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139707308507137,"checkpoint_ts":450139707308507137,"checkpoint_time":"2024-05-31 17:11:50.619"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [Fri May 31 17:11:56 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.6325063252.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:11:59 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1e5693b-6b05-4e31-8765-390860c26d5e {"id":"d1e5693b-6b05-4e31-8765-390860c26d5e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea870dc4 d1e5693b-6b05-4e31-8765-390860c26d5e /tidb/cdc/default/default/upstream/7375088515432773276 {"id":7375088515432773276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1e5693b-6b05-4e31-8765-390860c26d5e {"id":"d1e5693b-6b05-4e31-8765-390860c26d5e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea870dc4 d1e5693b-6b05-4e31-8765-390860c26d5e /tidb/cdc/default/default/upstream/7375088515432773276 {"id":7375088515432773276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1e5693b-6b05-4e31-8765-390860c26d5e {"id":"d1e5693b-6b05-4e31-8765-390860c26d5e","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdea870dc4 d1e5693b-6b05-4e31-8765-390860c26d5e /tidb/cdc/default/default/upstream/7375088515432773276 {"id":7375088515432773276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63291.out cli changefeed create --start-ts=450139693946503169 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375088515432773276,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:11:59.951840909+08:00","start_ts":450139693946503169,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139693946503169,"checkpoint_ts":450139693946503169,"checkpoint_time":"2024-05-31 17:10:59.647"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.simple not exists for 1-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139708652519433, "checkpoint_time": "2024-05-31 17:11:55.746", "error": null }' + echo '{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139708652519433, "checkpoint_time": "2024-05-31 17:11:55.746", "error": null }' { "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139708652519433, "checkpoint_time": "2024-05-31 17:11:55.746", "error": null } ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139708652519433, '"checkpoint_time":' '"2024-05-31' '17:11:55.746",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139708652519433, '"checkpoint_time":' '"2024-05-31' '17:11:55.746",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 table row_format.finish_mark not exists for 6-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63513.out cli changefeed --changefeed-id custom-changefeed-name pause PASS table row_format.finish_mark not exists for 7-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x table row_format.finish_mark not exists for 8-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null }' + echo '{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null }' { "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null } ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450139711470567427, '"checkpoint_time":' '"2024-05-31' '17:12:06.496",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450139711470567427, '"checkpoint_time":' '"2024-05-31' '17:12:06.496",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63578.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7375088515432773276,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:11:59.951840909+08:00","start_ts":450139693946503169,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139711470567427,"checkpoint_ts":450139711470567427,"checkpoint_time":"2024-05-31 17:12:06.496"} PASS table row_format.finish_mark not exists for 9-th check, retry later coverage: 3.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63618.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table row_format.finish_mark not exists for 10-th check, retry later table row_format.finish_mark exists check diff successfully + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null }' + echo '{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null }' { "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139711470567427, "checkpoint_time": "2024-05-31 17:12:06.496", "error": null } ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139711470567427, '"checkpoint_time":' '"2024-05-31' '17:12:06.496",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139711470567427, '"checkpoint_time":' '"2024-05-31' '17:12:06.496",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63703.out cli changefeed --changefeed-id custom-changefeed-name remove wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:12:16 CST 2024] <<<<<< run test case row_format success! >>>>>> Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 450139713829863429 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63764.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375088515432773276,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:12:20.98827539+08:00","start_ts":450139715232333826,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139715232333826,"checkpoint_ts":450139715232333826,"checkpoint_time":"2024-05-31 17:12:20.846"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [2024/05/31 17:12:20.746 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_1] [numQueries=200] [2024/05/31 17:12:21.665 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_4] [numQueries=200] [2024/05/31 17:12:22.671 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_3] [numQueries=200] [2024/05/31 17:12:23.162 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_5] [numQueries=200] [2024/05/31 17:12:24.661 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_8] [numQueries=200] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139715232333826, "checkpoint_time": "2024-05-31 17:12:20.846", "error": null }' + echo '{ "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139715232333826, "checkpoint_time": "2024-05-31 17:12:20.846", "error": null }' { "upstream_id": 7375088515432773276, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450139715232333826, "checkpoint_time": "2024-05-31 17:12:20.846", "error": null } ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139715232333826, '"checkpoint_time":' '"2024-05-31' '17:12:20.846",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375088515432773276, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450139715232333826, '"checkpoint_time":' '"2024-05-31' '17:12:20.846",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] [2024/05/31 17:12:25.675 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_2] [numQueries=200] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63848.out cli unsafe delete-service-gc-safepoint table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63866.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 ***************** properties ***************** "dotransactions"="false" "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="4" "operationcount"="0" "updateproportion"="0" "mysql.port"="4000" "scanproportion"="0" "mysql.user"="root" "insertproportion"="0" "recordcount"="20" "mysql.db"="changefeed_auto_stop_1" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" ********************************************** Run finished, takes 9.20604ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4201.8, Avg(us): 1765, Min(us): 889, Max(us): 4424, 95th(us): 5000, 99th(us): 5000 reset and all metadata truncated in PD! PASS [2024/05/31 17:12:26.677 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_0] [numQueries=200] [2024/05/31 17:12:26.678 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_9] [numQueries=200] coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later [2024/05/31 17:12:27.685 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_7] [numQueries=200] ***************** properties ***************** "threadcount"="4" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.port"="4000" "readproportion"="0" "mysql.db"="changefeed_auto_stop_2" "scanproportion"="0" "operationcount"="0" "dotransactions"="false" "insertproportion"="0" "workload"="core" "recordcount"="20" "readallfields"="true" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.55723ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3748.2, Avg(us): 1693, Min(us): 970, Max(us): 4452, 95th(us): 5000, 99th(us): 5000 [2024/05/31 17:12:28.672 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_6] [numQueries=200] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "readallfields"="true" "workload"="core" "mysql.db"="changefeed_auto_stop_3" "scanproportion"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "operationcount"="0" "threadcount"="4" "requestdistribution"="uniform" "updateproportion"="0" "readproportion"="0" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 9.473357ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4041.7, Avg(us): 1803, Min(us): 951, Max(us): 4546, 95th(us): 5000, 99th(us): 5000 table consistent_replicate_gbk.check1 not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later ***************** properties ***************** "threadcount"="4" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "insertproportion"="0" "readproportion"="0" "mysql.db"="changefeed_auto_stop_4" "readallfields"="true" "recordcount"="20" "workload"="core" "dotransactions"="false" "requestdistribution"="uniform" "operationcount"="0" ********************************************** Run finished, takes 8.813068ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4139.5, Avg(us): 1640, Min(us): 940, Max(us): 3911, 95th(us): 4000, 99th(us): 4000 [Fri May 31 17:12:30 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.3929939301.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63938.out cli unsafe resolve-lock --region=24 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.63966.out cli unsafe resolve-lock --region=24 --ts=450139716923162628 start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... PASS table foreign_key.finish_mark not exists for 3-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 91065 2577 --:--:-- --:--:-- --:--:-- 103k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" table consistent_replicate_gbk.check1 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:12:32 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4778847790.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:12:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:12:33 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.3935539357.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 4-th check, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 122k 0 --:--:-- --:--:-- --:--:-- 171k { "version": "v6.5.9-12-g7b0579a42", "git_hash": "7b0579a4218cd6fd5e5b4c65e4be42284f3b11b7", "id": "5334e42a-4f3f-46a3-a8e4-53acff93cebe", "pid": 63255, "is_owner": true }cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:12:35 CST 2024] <<<<<< run test case cli success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:12:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9a87f18b-d307-41bf-9b52-b2f4991963c1 {ClusterID:default ID:default/9a87f18b-d307-41bf-9b52-b2f4991963c1 Info:{"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} Status:0xc00496c408 TaskPositions:map[b4cc9106-a430-470b-893d-4b70dc3fd3dc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating tableID: 84, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4cc9106-a430-470b-893d-4b70dc3fd3dc {"id":"b4cc9106-a430-470b-893d-4b70dc3fd3dc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2de8 b4cc9106-a430-470b-893d-4b70dc3fd3dc /tidb/cdc/default/default/changefeed/info/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} /tidb/cdc/default/default/changefeed/status/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":450139716340940811,"min-table-barrier-ts":450139718818201601,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4cc9106-a430-470b-893d-4b70dc3fd3dc/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375088634849628583 {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9a87f18b-d307-41bf-9b52-b2f4991963c1 {ClusterID:default ID:default/9a87f18b-d307-41bf-9b52-b2f4991963c1 Info:{"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} Status:0xc00496c408 TaskPositions:map[b4cc9106-a430-470b-893d-4b70dc3fd3dc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating tableID: 84, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4cc9106-a430-470b-893d-4b70dc3fd3dc {"id":"b4cc9106-a430-470b-893d-4b70dc3fd3dc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2de8 b4cc9106-a430-470b-893d-4b70dc3fd3dc /tidb/cdc/default/default/changefeed/info/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"com+ grep -q 'failed to get info:' pression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} /tidb/cdc/default/default/changefeed/status/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":450139716340940811,"min-table-barrier-ts":450139718818201601,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4cc9106-a430-470b-893d-4b70dc3fd3dc/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375088634849628583 {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9a87f18b-d307-41bf-9b52-b2f4991963c1 {ClusterID:default ID:default/9a87f18b-d307-41bf-9b52-b2f4991963c1 Info:{"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} Status:0xc00496c408 TaskPositions:map[b4cc9106-a430-470b-893d-4b70dc3fd3dc:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating tableID: 84, resolvedTs: 450139716340940811, checkpointTs: 450139716340940811, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4cc9106-a430-470b-893d-4b70dc3fd3dc {"id":"b4cc9106-a430-470b-893d-4b70dc3fd3dc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaeb2de8 b4cc9106-a430-470b-893d-4b70dc3fd3dc /tidb/cdc/default/default/changefeed/info/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"upstream-id":7375088634849628583,"namespace":"default","changefeed-id":"9a87f18b-d307-41bf-9b52-b2f4991963c1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-31T17:11:29.456234573+08:00","start-ts":450139701739520002,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139701752627205} /tidb/cdc/default/default/changefeed/status/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":450139716340940811,"min-table-barrier-ts":450139718818201601,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4cc9106-a430-470b-893d-4b70dc3fd3dc/9a87f18b-d307-41bf-9b52-b2f4991963c1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375088634849628583 + grep -q 'etcd info' {"id":7375088634849628583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table foreign_key.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:12:36 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0fa8a78-e2df-4260-a07b-04bf003c39ab {"id":"c0fa8a78-e2df-4260-a07b-04bf003c39ab","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88779 c0fa8a78-e2df-4260-a07b-04bf003c39ab /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0fa8a78-e2df-4260-a07b-04bf003c39ab {"id":"c0fa8a78-e2df-4260-a07b-04bf003c39ab","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88779 c0fa8a78-e2df-4260-a07b-04bf003c39ab /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0fa8a78-e2df-4260-a07b-04bf003c39ab {"id":"c0fa8a78-e2df-4260-a07b-04bf003c39ab","address":"127.0.0.1:8302","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/capture/f297766f-b9de-41c8-a9ae-c92aac40c65f {"id":"f297766f-b9de-41c8-a9ae-c92aac40c65f","address":"127.0.0.1:8301","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88758 f297766f-b9de-41c8-a9ae-c92aac40c65f /tidb/cdc/default/__cdc_meta__/owner/22318fcdeaf88779 c0fa8a78-e2df-4260-a07b-04bf003c39ab /tidb/cdc/default/default/upstream/7375088658927533798 {"id":7375088658927533798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 68b1b6fb-8540-4299-bc76-f3fa8ace3b18 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=68b1b6fb-8540-4299-bc76-f3fa8ace3b18 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 68b1b6fb-8540-4299-bc76-f3fa8ace3b18 -s + info='{ "upstream_id": 7375088658927533798, "namespace": "default", "id": "68b1b6fb-8540-4299-bc76-f3fa8ace3b18", "state": "normal", "checkpoint_tso": 450139701979906049, "checkpoint_time": "2024-05-31 17:11:30.292", "error": null }' + echo '{ "upstream_id": 7375088658927533798, "namespace": "default", "id": "68b1b6fb-8540-4299-bc76-f3fa8ace3b18", "state": "normal", "checkpoint_tso": 450139701979906049, "checkpoint_time": "2024-05-31 17:11:30.292", "error": null }' { "upstream_id": 7375088658927533798, "namespace": "default", "id": "68b1b6fb-8540-4299-bc76-f3fa8ace3b18", "state": "normal", "checkpoint_tso": 450139701979906049, "checkpoint_time": "2024-05-31 17:11:30.292", "error": null } ++ echo '{' '"upstream_id":' 7375088658927533798, '"namespace":' '"default",' '"id":' '"68b1b6fb-8540-4299-bc76-f3fa8ace3b18",' '"state":' '"normal",' '"checkpoint_tso":' 450139701979906049, '"checkpoint_time":' '"2024-05-31' '17:11:30.292",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375088658927533798, '"namespace":' '"default",' '"id":' '"68b1b6fb-8540-4299-bc76-f3fa8ace3b18",' '"state":' '"normal",' '"checkpoint_tso":' 450139701979906049, '"checkpoint_time":' '"2024-05-31' '17:11:30.292",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 6-th check, retry later table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later [2024/05/31 17:12:37.268 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_1] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 7-th check, retry later [2024/05/31 17:12:40.447 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_4] table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later [2024/05/31 17:12:41.570 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_3] check_redo_resolved_ts 9a87f18b-d307-41bf-9b52-b2f4991963c1 450139720509030405 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/9a87f18b-d307-41bf-9b52-b2f4991963c1/meta global resolved ts 450139720391065602 not forward to 450139720509030405 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 8-th check, retry later [2024/05/31 17:12:42.679 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_5] check_redo_resolved_ts 9a87f18b-d307-41bf-9b52-b2f4991963c1 450139720509030405 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/9a87f18b-d307-41bf-9b52-b2f4991963c1/meta table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/05/31 17:12:43.766 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 17:12:43.767 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:12:43.770 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 17:12:43.770 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139719054131213] [resolvedTs=450139720915353603] [2024/05/31 17:12:43.771 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:12:43.771 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:12:43.771 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:12:43.771 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/31 17:12:43.774 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.707693ms] [2024/05/31 17:12:43.774 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=3.866802ms] [2024/05/31 17:12:43.774 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:12:43.777 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 17:12:43.778 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 17:12:43.778 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 17:12:43.778 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 17:12:43.781 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:12:43.783 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 17:12:43.783 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":450139719054131208,\"CommitTs\":450139719054131213,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/05/31 17:12:43.783 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":450139719054131208,\"CommitTs\":450139719054131213,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/05/31 17:12:43.784 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450139719054131208] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050: Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] [2024/05/31 17:12:43.985 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450139720915353603] [2024/05/31 17:12:43.985 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=450139720915353603] [2024/05/31 17:12:44.086 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=450139720915353603] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 17:12:44.086 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] Apply redo log successfully check diff successfully [Fri May 31 17:12:44 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT table foreign_key.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:12:44.059 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_8] table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later [2024/05/31 17:12:45.466 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_2] [2024/05/31 17:12:45.660 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_0] [2024/05/31 17:12:45.707 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_9] table foreign_key.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b106680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:52021, start at 2024-05-31 17:12:43.713327598 +0800 CST m=+5.927700149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [2024/05/31 17:12:46.178 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_7] [2024/05/31 17:12:46.262 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_6] TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:12:47 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> table test.end_mark_table not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table foreign_key.finish_mark not exists for 11-th check, retry later Starting CDC state checker... [Pipeline] } # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table foreign_key.finish_mark exists table test.end_mark_table not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // timeout [Pipeline] } + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:12:51 CST 2024] <<<<<< run test case foreign_key success! >>>>>> [Pipeline] // withEnv [Pipeline] } table test.end_mark_table not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 34933 1.5 0.0 1277408 123136 ? Sl 17:11 0:01 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717146705344806271.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 34937 1.3 0.0 1194968 132140 ? Sl 17:11 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717146705349225075.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 34973 1.6 0.0 1130008 144076 ? Sl 17:11 0:01 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717146708082102143.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ xargs kill -9 ++ awk '{print $2}' ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4850 0 --:--:-- --:--:-- --:--:-- 4959 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:12:51.968","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:12:46.669","now_ts":"2024-05-31 17:12:53.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:12:51.968","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:12:46.669","now_ts":"2024-05-31' '17:12:53.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:12:51.968","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:12:46.669","now_ts":"2024-05-31' '17:12:53.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table test.end_mark_table not exists for 4-th check, retry later table test.end_mark_table exists check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... + '[' 0 -eq 143 ']' + sleep 1 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:12:57 CST 2024] <<<<<< run test case processor_panic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + true + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.4460844610.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b291700005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:49015, start at 2024-05-31 17:13:08.960706836 +0800 CST m=+4.188240729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.967 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b2a0dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40472, start at 2024-05-31 17:13:09.958734992 +0800 CST m=+4.181554653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:13:10 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b106680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:52021, start at 2024-05-31 17:12:43.713327598 +0800 CST m=+5.927700149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b291700005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:49015, start at 2024-05-31 17:13:08.960706836 +0800 CST m=+4.188240729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.967 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b2a0dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40472, start at 2024-05-31 17:13:09.958734992 +0800 CST m=+4.181554653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fb0c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43765, start at 2024-05-31 17:13:32.140950683 +0800 CST m=+4.255080650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b105980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:51973, start at 2024-05-31 17:12:43.6368329 +0800 CST m=+5.975241013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b106680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--d2rt5, pid:52021, start at 2024-05-31 17:12:43.713327598 +0800 CST m=+5.927700149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:14:43.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:12:43.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:02:43.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.55892.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139736063868929 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139736063868929 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:13:40 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.5592155923.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:13:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e1486a-3a1a-4704-ad84-a97cda76fdc6 {"id":"12e1486a-3a1a-4704-ad84-a97cda76fdc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdecfa0c90 12e1486a-3a1a-4704-ad84-a97cda76fdc6 /tidb/cdc/default/default/upstream/7375089153239272424 {"id":7375089153239272424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e1486a-3a1a-4704-ad84-a97cda76fdc6 {"id":"12e1486a-3a1a-4704-ad84-a97cda76fdc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdecfa0c90 12e1486a-3a1a-4704-ad84-a97cda76fdc6 /tidb/cdc/default/default/upstream/7375089153239272424 {"id":7375089153239272424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12e1486a-3a1a-4704-ad84-a97cda76fdc6 {"id":"12e1486a-3a1a-4704-ad84-a97cda76fdc6","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdecfa0c90 12e1486a-3a1a-4704-ad84-a97cda76fdc6 /tidb/cdc/default/default/upstream/7375089153239272424 {"id":7375089153239272424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.55966.out cli changefeed create --start-ts=450139736063868929 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: cd6242d9-d22d-436b-b7c7-7ceaba9ac43a Info: {"upstream_id":7375089153239272424,"namespace":"default","id":"cd6242d9-d22d-436b-b7c7-7ceaba9ac43a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:13:44.488199767+08:00","start_ts":450139736063868929,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139736063868929,"checkpoint_ts":450139736063868929,"checkpoint_time":"2024-05-31 17:13:40.312"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b28e5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:48914, start at 2024-05-31 17:13:08.769927399 +0800 CST m=+4.046047079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b291700005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--3lx5z, pid:49015, start at 2024-05-31 17:13:08.960706836 +0800 CST m=+4.188240729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:08.967 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:08.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:08.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:13:51 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.5314253144.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b29ea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40392, start at 2024-05-31 17:13:09.837080657 +0800 CST m=+4.117867312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b2a0dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:40472, start at 2024-05-31 17:13:09.958734992 +0800 CST m=+4.181554653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:09.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:09.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:09.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.44408.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450139739227684866 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139739227684866 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:13:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27f896bd-f82e-41ab-921a-ba0b817a79a8 {"id":"27f896bd-f82e-41ab-921a-ba0b817a79a8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded21c590 27f896bd-f82e-41ab-921a-ba0b817a79a8 /tidb/cdc/default/default/upstream/7375089254528635294 {"id":7375089254528635294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27f896bd-f82e-41ab-921a-ba0b817a79a8 {"id":"27f896bd-f82e-41ab-921a-ba0b817a79a8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded21c590 27f896bd-f82e-41ab-921a-ba0b817a79a8 /tidb/cdc/default/default/upstream/7375089254528635294 {"id":7375089254528635294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27f896bd-f82e-41ab-921a-ba0b817a79a8 {"id":"27f896bd-f82e-41ab-921a-ba0b817a79a8","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded21c590 27f896bd-f82e-41ab-921a-ba0b817a79a8 /tidb/cdc/default/default/upstream/7375089254528635294 {"id":7375089254528635294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fb0c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43765, start at 2024-05-31 17:13:32.140950683 +0800 CST m=+4.255080650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b575240001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42201, start at 2024-05-31 17:13:56.305057085 +0800 CST m=+46.127138649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:56.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:56.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:56.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting CDC state checker... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fd900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43668, start at 2024-05-31 17:13:32.281876682 +0800 CST m=+4.453704663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b3fb0c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:43765, start at 2024-05-31 17:13:32.140950683 +0800 CST m=+4.255080650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:32.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:32.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:32.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed skip some DDLs in tidb v4.0.x [Fri May 31 17:14:17 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.4777547777.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:14:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82f82d7c-28af-474c-a5cd-01f93b92df6c {"id":"82f82d7c-28af-474c-a5cd-01f93b92df6c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded84f688 82f82d7c-28af-474c-a5cd-01f93b92df6c /tidb/cdc/default/default/upstream/7375089362030892956 {"id":7375089362030892956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82f82d7c-28af-474c-a5cd-01f93b92df6c {"id":"82f82d7c-28af-474c-a5cd-01f93b92df6c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded84f688 82f82d7c-28af-474c-a5cd-01f93b92df6c /tidb/cdc/default/default/upstream/7375089362030892956 {"id":7375089362030892956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82f82d7c-28af-474c-a5cd-01f93b92df6c {"id":"82f82d7c-28af-474c-a5cd-01f93b92df6c","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded84f688 82f82d7c-28af-474c-a5cd-01f93b92df6c /tidb/cdc/default/default/upstream/7375089362030892956 {"id":7375089362030892956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b575240001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42201, start at 2024-05-31 17:13:56.305057085 +0800 CST m=+46.127138649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:56.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:56.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:56.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table savepoint.finish_mark not exists for 1-th check, retry later table savepoint.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... table savepoint.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:14:43 CST 2024] <<<<<< run test case savepoint success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "workload"="core" "readproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.user"="root" "mysql.port"="4000" "dotransactions"="false" "scanproportion"="0" "updateproportion"="0" "fieldcount"="3" "recordcount"="50000" "insertproportion"="0" "mysql.db"="consistent_replicate_nfs" "threadcount"="10" "requestdistribution"="uniform" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b5d7900001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42106, start at 2024-05-31 17:14:02.633248858 +0800 CST m=+52.511187618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:16:02.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:14:02.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:04:02.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37b575240001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--4h82c, pid:42201, start at 2024-05-31 17:13:56.305057085 +0800 CST m=+46.127138649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:15:56.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:13:56.747 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:03:56.747 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri May 31 17:14:52 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.4669746699.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:14:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2467a6a2-491c-441a-8cf1-33768545d8e4 {"id":"2467a6a2-491c-441a-8cf1-33768545d8e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdee0dca8d 2467a6a2-491c-441a-8cf1-33768545d8e4 /tidb/cdc/default/default/upstream/7375089294862767591 {"id":7375089294862767591,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2467a6a2-491c-441a-8cf1-33768545d8e4 {"id":"2467a6a2-491c-441a-8cf1-33768545d8e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdee0dca8d 2467a6a2-491c-441a-8cf1-33768545d8e4 /tidb/cdc/default/default/upstream/7375089294862767591 {"id":7375089294862767591,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2467a6a2-491c-441a-8cf1-33768545d8e4 {"id":"2467a6a2-491c-441a-8cf1-33768545d8e4","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdee0dca8d 2467a6a2-491c-441a-8cf1-33768545d8e4 /tidb/cdc/default/default/upstream/7375089294862767591 {"id":7375089294862767591,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 INSERT - Takes(s): 10.0, Count: 28107, OPS: 2812.1, Avg(us): 3530, Min(us): 638, Max(us): 884243, 95th(us): 3000, 99th(us): 83000 ***************** properties ***************** "mysql.user"="root" "insertproportion"="0" "readallfields"="true" "recordcount"="10" "scanproportion"="0" "dotransactions"="false" "workload"="core" "updateproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "threadcount"="2" "mysql.db"="changefeed_dup_error_restart" "operationcount"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 7.648003ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2422.5, Avg(us): 1476, Min(us): 866, Max(us): 3521, 95th(us): 4000, 99th(us): 4000 [Fri May 31 17:14:50 CST 2024] <<<<<< START cdc server in changefeed_dup_error_restart case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLDuplicateEntryError=5%return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.4450944511.out server --log-file /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:14:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b49332c9-3596-4f37-a3ac-4ec06f862f86 {"id":"b49332c9-3596-4f37-a3ac-4ec06f862f86","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded252016 b49332c9-3596-4f37-a3ac-4ec06f862f86 /tidb/cdc/default/default/upstream/7375089262908627637 {"id":7375089262908627637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b49332c9-3596-4f37-a3ac-4ec06f862f86 {"id":"b49332c9-3596-4f37-a3ac-4ec06f862f86","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded252016 b49332c9-3596-4f37-a3ac-4ec06f862f86 /tidb/cdc/default/default/upstream/7375089262908627637 {"id":7375089262908627637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b49332c9-3596-4f37-a3ac-4ec06f862f86 {"id":"b49332c9-3596-4f37-a3ac-4ec06f862f86","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcded252016 b49332c9-3596-4f37-a3ac-4ec06f862f86 /tidb/cdc/default/default/upstream/7375089262908627637 {"id":7375089262908627637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.44569.out cli changefeed create --start-ts=450139739227684866 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: e3911493-a25f-49a1-aef3-b1688f3ae1e9 Info: {"upstream_id":7375089262908627637,"namespace":"default","id":"e3911493-a25f-49a1-aef3-b1688f3ae1e9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:14:53.659721009+08:00","start_ts":450139739227684866,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139739227684866,"checkpoint_ts":450139739227684866,"checkpoint_time":"2024-05-31 17:13:52.381"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x INSERT - Takes(s): 20.0, Count: 43264, OPS: 2161.1, Avg(us): 4592, Min(us): 638, Max(us): 884243, 95th(us): 4000, 99th(us): 86000 Run finished, takes 24.438049254s INSERT - Takes(s): 24.4, Count: 50000, OPS: 2046.4, Avg(us): 4820, Min(us): 638, Max(us): 884243, 95th(us): 5000, 99th(us): 86000 table consistent_replicate_nfs.usertable exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3773 0 --:--:-- --:--:-- --:--:-- 3810 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:15:02.119","puller_resolved_ts":"2024-05-31 17:14:55.918","last_synced_ts":"2024-05-31 17:12:46.669","now_ts":"2024-05-31 17:15:03.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:15:02.119","puller_resolved_ts":"2024-05-31' '17:14:55.918","last_synced_ts":"2024-05-31' '17:12:46.669","now_ts":"2024-05-31' '17:15:03.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:15:02.119","puller_resolved_ts":"2024-05-31' '17:14:55.918","last_synced_ts":"2024-05-31' '17:12:46.669","now_ts":"2024-05-31' '17:15:03.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 changefeed is working normally rts: 450139760732667900->450139761256955900 checkpoint: 450139760994811900->450139761256955900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/31 17:15:15.568 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=84.867513ms] [namespace=] [changefeed=] run task successfully resume changefeed ba89b27b-3988-4ca6-98ad-0302b9be3f71 from 450139760942383113 check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... changefeed is working normally rts: 450139761519099900->450139762043387900 checkpoint: 450139760942383100->450139762043387900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/31 17:15:18.540 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450139760942383108] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=] [changefeed=] [error="Error 1007: Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "450139759502426194" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" changefeed is working normally rts: 450139762318639100->450139762331484160 checkpoint: 450139762318639100->450139762331484160 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/31 17:15:21.615 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=84.748756ms] [namespace=] [changefeed=] run task successfully [2024/05/31 17:15:23.279 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/05/31 17:15:23.279 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/31 17:15:23.279 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=450139757772537976] [resolvedTs=450139759502426194] [2024/05/31 17:15:23.280 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=309.212µs] [2024/05/31 17:15:23.280 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:15:23.280 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:15:23.286 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:15:23.288 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/31 17:15:23.290 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/31 17:15:23.290 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/31 17:15:23.290 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/05/31 17:15:23.294 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/31 17:15:23.298 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/31 17:15:23.309 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=29.557522ms] resume changefeed ba89b27b-3988-4ca6-98ad-0302b9be3f71 from 450139762331484177 check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:15:24.978 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=82] [checkpointTs=450139759502426194] [2024/05/31 17:15:24.978 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=82] [checkpointTs=450139759502426194] [2024/05/31 17:15:25.079 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=10042] [appliedDDLCount=0] [currentCheckpoint=450139759502426194] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/31 17:15:25.079 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] Apply redo log successfully check diff successfully [Fri May 31 17:15:25 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 45774 run task failed 1-th time, retry later table changefeed_dup_error_restart.finish_mark_1 exists check diff successfully ***************** properties ***************** "insertproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "readproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.db"="changefeed_dup_error_restart" "mysql.port"="4000" "recordcount"="10" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "readallfields"="true" "operationcount"="0" "dotransactions"="false" ********************************************** Run finished, takes 3.597218ms INSERT - Takes(s): 0.0, Count: 10, OPS: 4423.1, Avg(us): 671, Min(us): 417, Max(us): 1487, 95th(us): 2000, 99th(us): 2000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed is working normally rts: 450139764153647100->450139764677935100 checkpoint: 450139764153647100->450139764677935100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/31 17:15:25.533 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450139762331484169] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed is working normally rts: 450139764677935100->450139765202223100 checkpoint: 450139764677935100->450139764887388160 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/31 17:15:31.762 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=82.500465ms] [namespace=] [changefeed=] run task successfully resume changefeed ba89b27b-3988-4ca6-98ad-0302b9be3f71 from 450139764887388173 check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36204, start at 2024-05-31 17:15:30.4464554 +0800 CST m=+5.206657716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } run task failed 1-th time, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 changefeed is working normally rts: 450139766775087100->450139767037231100 checkpoint: 450139766775087100->450139767299375100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/31 17:15:33.591 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=450139764887388168] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 changefeed is working normally rts: 450139767299375100->450139767823663100 checkpoint: 450139767299375100->450139767823663100 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/31 17:15:40.882 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=150.480253ms] [namespace=] [changefeed=] run task successfully resume changefeed ba89b27b-3988-4ca6-98ad-0302b9be3f71 from 450139767443554317 check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory run task failed 1-th time, retry later check_ts_forward ba89b27b-3988-4ca6-98ad-0302b9be3f71 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... changefeed is working normally rts: 450139769134383100->450139769658408960 checkpoint: 450139769396527100->450139769658408960 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/31 17:15:43.730 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=143.477142ms] [namespace=] [changefeed=] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:15:50 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36204, start at 2024-05-31 17:15:30.4464554 +0800 CST m=+5.206657716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table changefeed_dup_error_restart.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:15:54 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7375089294862767591 450139771350810626 450139755779194884 run task successfully check_changefeed_state http://127.0.0.1:2379 b5a29506-643d-42fa-9996-ae15e9b566ca stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=b5a29506-643d-42fa-9996-ae15e9b566ca + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b5a29506-643d-42fa-9996-ae15e9b566ca -s + info='{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' + echo '{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' { "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null } ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"stopped",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"stopped",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7375089294862767591 run task successfully check_changefeed_state http://127.0.0.1:2379 b5a29506-643d-42fa-9996-ae15e9b566ca normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=b5a29506-643d-42fa-9996-ae15e9b566ca + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b5a29506-643d-42fa-9996-ae15e9b566ca -s + info='{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "normal", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' + echo '{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "normal", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' { "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "normal", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null } ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"normal",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"normal",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7375089294862767591 450139771874836482 450139771874836483 run task successfully check_changefeed_state http://127.0.0.1:2379 b5a29506-643d-42fa-9996-ae15e9b566ca stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=b5a29506-643d-42fa-9996-ae15e9b566ca + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b5a29506-643d-42fa-9996-ae15e9b566ca -s + info='{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' + echo '{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null }' { "upstream_id": 7375089294862767591, "namespace": "default", "id": "b5a29506-643d-42fa-9996-ae15e9b566ca", "state": "stopped", "checkpoint_tso": 450139771874836483, "checkpoint_time": "2024-05-31 17:15:56.920", "error": null } ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"stopped",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"b5a29506-643d-42fa-9996-ae15e9b566ca",' '"state":' '"stopped",' '"checkpoint_tso":' 450139771874836483, '"checkpoint_time":' '"2024-05-31' '17:15:56.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 941f753c-6069-4d4b-b4c8-d462dc4698c3 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=941f753c-6069-4d4b-b4c8-d462dc4698c3 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 941f753c-6069-4d4b-b4c8-d462dc4698c3 -s + info='{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "941f753c-6069-4d4b-b4c8-d462dc4698c3", "state": "normal", "checkpoint_tso": 450139773460807682, "checkpoint_time": "2024-05-31 17:16:02.970", "error": null }' + echo '{ "upstream_id": 7375089294862767591, "namespace": "default", "id": "941f753c-6069-4d4b-b4c8-d462dc4698c3", "state": "normal", "checkpoint_tso": 450139773460807682, "checkpoint_time": "2024-05-31 17:16:02.970", "error": null }' { "upstream_id": 7375089294862767591, "namespace": "default", "id": "941f753c-6069-4d4b-b4c8-d462dc4698c3", "state": "normal", "checkpoint_tso": 450139773460807682, "checkpoint_time": "2024-05-31 17:16:02.970", "error": null } ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"941f753c-6069-4d4b-b4c8-d462dc4698c3",' '"state":' '"normal",' '"checkpoint_tso":' 450139773460807682, '"checkpoint_time":' '"2024-05-31' '17:16:02.970",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375089294862767591, '"namespace":' '"default",' '"id":' '"941f753c-6069-4d4b-b4c8-d462dc4698c3",' '"state":' '"normal",' '"checkpoint_tso":' 450139773460807682, '"checkpoint_time":' '"2024-05-31' '17:16:02.970",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7375089294862767591 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory run task successfully Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Changefeed remove successfully. ID: b5a29506-643d-42fa-9996-ae15e9b566ca CheckpointTs: 450139771874836483 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_safepoint_forward http://127.0.0.1:2379 7375089294862767591 450139771874836482 450139773460807682 450139771874836483 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... run task successfully chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Changefeed remove successfully. ID: 941f753c-6069-4d4b-b4c8-d462dc4698c3 CheckpointTs: 450139774758420484 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7375089294862767591 run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri May 31 17:16:09 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb2400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51633, start at 2024-05-31 17:16:11.298225511 +0800 CST m=+4.285572314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36131, start at 2024-05-31 17:15:30.46250996 +0800 CST m=+5.314631398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bb33a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:36204, start at 2024-05-31 17:15:30.4464554 +0800 CST m=+5.206657716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:17:30.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:15:30.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:05:30.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.39826.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be0ba00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45666, start at 2024-05-31 17:16:17.037137608 +0800 CST m=+4.243657441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:17.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:17.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:17.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log + set +x + tso='450139777559953409 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139777559953409 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139777559953409 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:16:19 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3985639858.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:16:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eadc616a-f41f-4814-a676-6f2178ccf16f {"id":"eadc616a-f41f-4814-a676-6f2178ccf16f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdef60d890 eadc616a-f41f-4814-a676-6f2178ccf16f /tidb/cdc/default/default/upstream/7375089875742879160 {"id":7375089875742879160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eadc616a-f41f-4814-a676-6f2178ccf16f {"id":"eadc616a-f41f-4814-a676-6f2178ccf16f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdef60d890 eadc616a-f41f-4814-a676-6f2178ccf16f /tidb/cdc/default/default/upstream/7375089875742879160 {"id":7375089875742879160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eadc616a-f41f-4814-a676-6f2178ccf16f {"id":"eadc616a-f41f-4814-a676-6f2178ccf16f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdef60d890 eadc616a-f41f-4814-a676-6f2178ccf16f /tidb/cdc/default/default/upstream/7375089875742879160 {"id":7375089875742879160,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139777559953409 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.39900.out cli changefeed create --start-ts=450139777559953409 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375089875742879160,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:16:22.630142392+08:00","start_ts":450139777559953409,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139777559953409,"checkpoint_ts":450139777559953409,"checkpoint_time":"2024-05-31 17:16:18.607"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb2400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51633, start at 2024-05-31 17:16:11.298225511 +0800 CST m=+4.285572314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be0ba00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45666, start at 2024-05-31 17:16:17.037137608 +0800 CST m=+4.243657441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:17.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:17.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:17.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 17821 0 --:--:-- --:--:-- --:--:-- 18075 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:16:18.607","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 17:16:43.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:16:18.607","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:16:43.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:16:18.607","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:16:43.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb0b40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51536, start at 2024-05-31 17:16:11.216497548 +0800 CST m=+4.253690384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37bdb2400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--xj498, pid:51633, start at 2024-05-31 17:16:11.298225511 +0800 CST m=+4.285572314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:11.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:11.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:11.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.55698.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139787325865985 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139787325865985 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:16:56 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.5574455746.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:16:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67c0d799-ca69-4339-b990-83c641a90b9b {"id":"67c0d799-ca69-4339-b990-83c641a90b9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdefefe18e 67c0d799-ca69-4339-b990-83c641a90b9b /tidb/cdc/default/default/upstream/7375090049628917407 {"id":7375090049628917407,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67c0d799-ca69-4339-b990-83c641a90b9b {"id":"67c0d799-ca69-4339-b990-83c641a90b9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdefefe18e 67c0d799-ca69-4339-b990-83c641a90b9b /tidb/cdc/default/default/upstream/7375090049628917407 {"id":7375090049628917407,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67c0d799-ca69-4339-b990-83c641a90b9b {"id":"67c0d799-ca69-4339-b990-83c641a90b9b","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdefefe18e 67c0d799-ca69-4339-b990-83c641a90b9b /tidb/cdc/default/default/upstream/7375090049628917407 {"id":7375090049628917407,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.55815.out cli changefeed create --start-ts=450139787325865985 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a80ea64d-f014-4376-b1f3-e64fc75b7b44 Info: {"upstream_id":7375090049628917407,"namespace":"default","id":"a80ea64d-f014-4376-b1f3-e64fc75b7b44","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:16:59.553077247+08:00","start_ts":450139787325865985,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139787325865985,"checkpoint_ts":450139787325865985,"checkpoint_time":"2024-05-31 17:16:55.861"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be09600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45575, start at 2024-05-31 17:16:16.887145554 +0800 CST m=+4.140541780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:16.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:16.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:16.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37be0ba00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--0wrgd, pid:45666, start at 2024-05-31 17:16:17.037137608 +0800 CST m=+4.243657441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:18:17.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:16:17.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:06:17.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.49575.out cli tso query --pd=http://127.0.0.1:2379 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... + set +x + tso='450139788790464513 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139788790464513 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c1805c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40833, start at 2024-05-31 17:17:13.665950336 +0800 CST m=+5.917110341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c1805c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40833, start at 2024-05-31 17:17:13.665950336 +0800 CST m=+5.917110341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_sequence.finish_mark not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_sequence.finish_mark not exists for 4-th check, retry later table ddl_sequence.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:18:08 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> ***************** properties ***************** "readallfields"="true" "threadcount"="4" "workload"="core" "mysql.user"="root" "dotransactions"="false" "updateproportion"="0" "scanproportion"="0" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" "recordcount"="20" "mysql.db"="changefeed_error" "insertproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.415578ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3587.6, Avg(us): 1967, Min(us): 1110, Max(us): 4859, 95th(us): 5000, 99th(us): 5000 [Fri May 31 17:17:59 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.4968749689.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 {"id":"3b4ea9b0-a9ab-4693-9443-5634cbc5a015","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b81a 3b4ea9b0-a9ab-4693-9443-5634cbc5a015 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 {"id":"3b4ea9b0-a9ab-4693-9443-5634cbc5a015","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b81a 3b4ea9b0-a9ab-4693-9443-5634cbc5a015 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 {"id":"3b4ea9b0-a9ab-4693-9443-5634cbc5a015","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b81a 3b4ea9b0-a9ab-4693-9443-5634cbc5a015 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.49750.out cli changefeed create --start-ts=450139788790464513 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7375090075179853152,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:18:02.771792039+08:00","start_ts":450139788790464513,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513,"checkpoint_time":"2024-05-31 17:17:01.448"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450139788790464513, "checkpoint_time": "2024-05-31 17:17:01.448", "error": { "time": "2024-05-31T17:18:02.840559163+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at 450139788790464513" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450139788790464513, "checkpoint_time": "2024-05-31 17:17:01.448", "error": { "time": "2024-05-31T17:18:02.840559163+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at 450139788790464513" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450139788790464513, "checkpoint_time": "2024-05-31 17:17:01.448", "error": { "time": "2024-05-31T17:18:02.840559163+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at 450139788790464513" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450139788790464513, '"checkpoint_time":' '"2024-05-31' '17:17:01.448",' '"error":' '{' '"time":' '"2024-05-31T17:18:02.840559163+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at '450139788790464513"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450139788790464513, '"checkpoint_time":' '"2024-05-31' '17:17:01.448",' '"error":' '{' '"time":' '"2024-05-31T17:18:02.840559163+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at '450139788790464513"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at 450139788790464513' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450139788790464213 is earlier than or equal to GC safepoint at 450139788790464513 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.49845.out cli changefeed resume -c changefeed-error PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table changefeed_error.usertable not exists for 1-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "readproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "readallfields"="true" "operationcount"="0" "mysql.db"="changefeed_error" "insertproportion"="0" "recordcount"="20" "mysql.port"="4000" "scanproportion"="0" "threadcount"="4" "updateproportion"="0" "mysql.user"="root" "workload"="core" ********************************************** Run finished, takes 3.778861ms INSERT - Takes(s): 0.0, Count: 20, OPS: 7657.6, Avg(us): 686, Min(us): 422, Max(us): 1547, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"3b4ea9b0-a9ab-4693-9443-5634cbc5a015","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + [[ /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c17ec40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40778, start at 2024-05-31 17:17:13.521957401 +0800 CST m=+5.868657202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37c1805c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:40833, start at 2024-05-31 17:17:13.665950336 +0800 CST m=+5.917110341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:19:13.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:17:13.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:07:13.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + [[ /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.44417.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139806935547905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139806935547905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139806935547905 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:18:10 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4445144453.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d3bd2da-fce2-4a63-804c-ab5aeb653868 {"id":"9d3bd2da-fce2-4a63-804c-ab5aeb653868","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf1176690 9d3bd2da-fce2-4a63-804c-ab5aeb653868 /tidb/cdc/default/default/upstream/7375090303357962780 {"id":7375090303357962780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d3bd2da-fce2-4a63-804c-ab5aeb653868 {"id":"9d3bd2da-fce2-4a63-804c-ab5aeb653868","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf1176690 9d3bd2da-fce2-4a63-804c-ab5aeb653868 /tidb/cdc/default/default/upstream/7375090303357962780 {"id":7375090303357962780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d3bd2da-fce2-4a63-804c-ab5aeb653868 {"id":"9d3bd2da-fce2-4a63-804c-ab5aeb653868","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf1176690 9d3bd2da-fce2-4a63-804c-ab5aeb653868 /tidb/cdc/default/default/upstream/7375090303357962780 {"id":7375090303357962780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139806935547905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.44499.out cli changefeed create --start-ts=450139806935547905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375090303357962780,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:18:14.548337531+08:00","start_ts":450139806935547905,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139806935547905,"checkpoint_ts":450139806935547905,"checkpoint_time":"2024-05-31 17:18:10.666"} PASS check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + [[ /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3b4ea9b0-a9ab-4693-9443-5634cbc5a015 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4591 0 --:--:-- --:--:-- --:--:-- 4604 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:18:10.666","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 17:18:14.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:18:10.666","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:18:14.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:18:10.666","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:18:14.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-31 17:18:10.666' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:18:10.666","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:18:14.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:18:10.666","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:18:14.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-31 17:18:15' + echo 'sink_checkpoint_ts is 2024-05-31' 17:18:10.666 sink_checkpoint_ts is 2024-05-31 17:18:10.666 ++ date -d '2024-05-31 17:18:10.666' +%s + checkpoint_timestamp=1717147090 ++ date -d '2024-05-31 17:18:15' +%s + current_timestamp=1717147095 + '[' 5 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:18:20 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5008050082.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96e51c5d-5979-4e58-946f-685906d872bc {"id":"96e51c5d-5979-4e58-946f-685906d872bc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b88b 96e51c5d-5979-4e58-946f-685906d872bc /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375090075179853152,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:18:02.771792039+08:00","start-ts":450139788790464513,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-31T17:18:20.726471795+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139809565376517} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450139805842931718,"min-table-barrier-ts":450139805842931718,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96e51c5d-5979-4e58-946f-685906d872bc {"id":"96e51c5d-5979-4e58-946f-685906d872bc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b88b 96e51c5d-5979-4e58-946f-685906d872bc /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375090075179853152,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:18:02.771792039+08:00","start-ts":450139788790464513,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-31T17:18:20.726471795+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139809565376517} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450139805842931718,"min-table-barrier-ts":450139805842931718,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96e51c5d-5979-4e58-946f-685906d872bc {"id":"96e51c5d-5979-4e58-946f-685906d872bc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b88b 96e51c5d-5979-4e58-946f-685906d872bc /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375090075179853152,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T17:18:02.771792039+08:00","start-ts":450139788790464513,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-31T17:18:20.726471795+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-12-g7b0579a42","epoch":450139809565376517} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450139805842931718,"min-table-barrier-ts":450139805842931718,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450139805842931718, "checkpoint_time": "2024-05-31 17:18:06.498", "error": { "time": "2024-05-31T17:18:20.726471795+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450139805842931718, "checkpoint_time": "2024-05-31 17:18:06.498", "error": { "time": "2024-05-31T17:18:20.726471795+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450139805842931718, "checkpoint_time": "2024-05-31 17:18:06.498", "error": { "time": "2024-05-31T17:18:20.726471795+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450139805842931718, '"checkpoint_time":' '"2024-05-31' '17:18:06.498",' '"error":' '{' '"time":' '"2024-05-31T17:18:20.726471795+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450139805842931718, '"checkpoint_time":' '"2024-05-31' '17:18:06.498",' '"error":' '{' '"time":' '"2024-05-31T17:18:20.726471795+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50180.out cli changefeed remove -c changefeed-error Changefeed remove successfully. ID: changefeed-error CheckpointTs: 450139805842931718 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:18:25 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5026450266.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:28 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c87067d-b1ab-4afd-92a9-8aecff149a0f {"id":"0c87067d-b1ab-4afd-92a9-8aecff149a0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b8ae 0c87067d-b1ab-4afd-92a9-8aecff149a0f /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c87067d-b1ab-4afd-92a9-8aecff149a0f {"id":"0c87067d-b1ab-4afd-92a9-8aecff149a0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b8ae 0c87067d-b1ab-4afd-92a9-8aecff149a0f /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c87067d-b1ab-4afd-92a9-8aecff149a0f {"id":"0c87067d-b1ab-4afd-92a9-8aecff149a0f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b8ae 0c87067d-b1ab-4afd-92a9-8aecff149a0f /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50313.out cli changefeed create --start-ts=450139788790464513 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7375090075179853152,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:18:28.454753389+08:00","start_ts":450139788790464513,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513,"checkpoint_time":"2024-05-31 17:17:01.448"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1392 0 --:--:-- --:--:-- --:--:-- 1409 + info='{"state":"normal","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513}' + echo '{"state":"normal","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513}' {"state":"normal","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513} ++ echo '{"state":"normal","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 4022 0 --:--:-- --:--:-- --:--:-- 4080 + info='{"state":"warning","resolved_ts":450139803929280540,"checkpoint_ts":450139803929280540,"last_warning":{"time":"2024-05-31T17:18:29.733741116+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":450139803929280540,"checkpoint_ts":450139803929280540,"last_warning":{"time":"2024-05-31T17:18:29.733741116+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":450139803929280540,"checkpoint_ts":450139803929280540,"last_warning":{"time":"2024-05-31T17:18:29.733741116+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":450139803929280540,"checkpoint_ts":450139803929280540,"last_warning":{"time":"2024-05-31T17:18:29.733741116+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":450139803929280540,"checkpoint_ts":450139803929280540,"last_warning":{"time":"2024-05-31T17:18:29.733741116+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50377.out cli changefeed remove -c changefeed-error-1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 450139803929280540 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:18:32 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5043150433.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:35 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61e00e9c-000e-4d7d-9f30-27e603fc3543 {"id":"61e00e9c-000e-4d7d-9f30-27e603fc3543","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b903 61e00e9c-000e-4d7d-9f30-27e603fc3543 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61e00e9c-000e-4d7d-9f30-27e603fc3543 {"id":"61e00e9c-000e-4d7d-9f30-27e603fc3543","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b903 61e00e9c-000e-4d7d-9f30-27e603fc3543 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61e00e9c-000e-4d7d-9f30-27e603fc3543 {"id":"61e00e9c-000e-4d7d-9f30-27e603fc3543","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b903 61e00e9c-000e-4d7d-9f30-27e603fc3543 /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50485.out cli changefeed create --start-ts=450139788790464513 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7375090075179853152,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:18:35.887103949+08:00","start_ts":450139788790464513,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139788790464513,"checkpoint_ts":450139788790464513,"checkpoint_time":"2024-05-31 17:17:01.448"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T17:18:35.892554969+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T17:18:35.892554969+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-05-31T17:18:35.892554969+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-31T17:18:35.892554969+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-05-31T17:18:35.892554969+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450139788790464513 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50558.out cli changefeed remove -c changefeed-error-2 Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:18:37 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5060550607.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:18:40 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9a312d2-5f5e-4093-b39f-39ec6de3050d {"id":"e9a312d2-5f5e-4093-b39f-39ec6de3050d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b926 e9a312d2-5f5e-4093-b39f-39ec6de3050d /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9a312d2-5f5e-4093-b39f-39ec6de3050d {"id":"e9a312d2-5f5e-4093-b39f-39ec6de3050d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b926 e9a312d2-5f5e-4093-b39f-39ec6de3050d /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9a312d2-5f5e-4093-b39f-39ec6de3050d {"id":"e9a312d2-5f5e-4093-b39f-39ec6de3050d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf005b926 e9a312d2-5f5e-4093-b39f-39ec6de3050d /tidb/cdc/default/default/upstream/7375090075179853152 {"id":7375090075179853152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50660.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/conf/enable_redo.toml Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7375090075179853152,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:18:41.122008674+08:00","start_ts":450139814887161861,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/changefeed_error/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139814887161861,"checkpoint_ts":450139814887161861,"checkpoint_time":"2024-05-31 17:18:40.999"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450139814887161861, '"checkpoint_time":' '"2024-05-31' '17:18:40.999",' '"error":' '{' '"time":' '"2024-05-31T17:18:41.209771112+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450139814887161861, "checkpoint_time": "2024-05-31 17:18:40.999", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450139814887161861, '"checkpoint_time":' '"2024-05-31' '17:18:40.999",' '"error":' '{' '"time":' '"2024-05-31T17:18:41.209771112+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null } ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50828.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": { "time": "2024-05-31T17:18:41.209771112+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' '{' '"time":' '"2024-05-31T17:18:41.209771112+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' '{' '"time":' '"2024-05-31T17:18:41.209771112+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50910.out cli changefeed resume -c changefeed-initialize-error PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null }' + echo '{ "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null }' { "upstream_id": 7375090075179853152, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450139816341798917, "checkpoint_time": "2024-05-31 17:18:46.548", "error": null } ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375090075179853152, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450139816341798917, '"checkpoint_time":' '"2024-05-31' '17:18:46.548",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.50990.out cli changefeed remove -c changefeed-initialize-error Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 450139816341798917 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:18:51 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4227 0 --:--:-- --:--:-- --:--:-- 4263 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:19:14.515","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:19:08.566","now_ts":"2024-05-31 17:19:15.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:19:14.515","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:19:08.566","now_ts":"2024-05-31' '17:19:15.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:19:14.515","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:19:08.566","now_ts":"2024-05-31' '17:19:15.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5510 0 --:--:-- --:--:-- --:--:-- 5525 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:21:23.616","puller_resolved_ts":"2024-05-31 17:21:15.616","last_synced_ts":"2024-05-31 17:19:08.566","now_ts":"2024-05-31 17:21:25.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:21:23.616","puller_resolved_ts":"2024-05-31' '17:21:15.616","last_synced_ts":"2024-05-31' '17:19:08.566","now_ts":"2024-05-31' '17:21:25.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 43474 5.8 0.0 13464280 137824 ? Sl 17:17 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 43529 3.3 0.0 13323924 137048 ? Sl 17:18 0:06 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-31T17:21:33.284+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:33.288+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:33.394+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:39.286+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:39.289+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:39.395+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:43.271+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:21:43.271+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:21:43.272+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:21:43.272+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:21:43.382+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:21:43.382+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717147303.4683676,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0005fd880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717147303.4684293,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717147303.4743881,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0015fcfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717147303.4744408,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717147303.5388808,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00165a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717147303.5389402,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:21:45.287+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:45.290+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:45.396+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:21:51.289+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:51.291+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:51.397+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:21:57.290+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:57.292+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:21:57.398+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:22:03.291+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:03.293+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:03.399+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:22:09.293+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:09.294+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:09.400+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:22:15.294+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000924a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:15.295+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00092b6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:22:15.401+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ac2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d562440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45762, start at 2024-05-31 17:22:39.401772061 +0800 CST m=+4.268093579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d562440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45762, start at 2024-05-31 17:22:39.401772061 +0800 CST m=+4.268093579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d55e54001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45677, start at 2024-05-31 17:22:39.160127817 +0800 CST m=+4.109349944 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37d562440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:45762, start at 2024-05-31 17:22:39.401772061 +0800 CST m=+4.268093579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:24:39.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:22:39.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:12:39.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.49801.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139889848025089 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139889848025089 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139889848025089 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:23:27 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4984149843.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:23:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76253c5d-b2d8-457e-aeb2-8f1c48bc663f {"id":"76253c5d-b2d8-457e-aeb2-8f1c48bc663f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf5ed728d 76253c5d-b2d8-457e-aeb2-8f1c48bc663f /tidb/cdc/default/default/upstream/7375091709139981782 {"id":7375091709139981782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76253c5d-b2d8-457e-aeb2-8f1c48bc663f {"id":"76253c5d-b2d8-457e-aeb2-8f1c48bc663f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf5ed728d 76253c5d-b2d8-457e-aeb2-8f1c48bc663f /tidb/cdc/default/default/upstream/7375091709139981782 {"id":7375091709139981782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76253c5d-b2d8-457e-aeb2-8f1c48bc663f {"id":"76253c5d-b2d8-457e-aeb2-8f1c48bc663f","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdf5ed728d 76253c5d-b2d8-457e-aeb2-8f1c48bc663f /tidb/cdc/default/default/upstream/7375091709139981782 {"id":7375091709139981782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139889848025089 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.49901.out cli changefeed create --start-ts=450139889848025089 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375091709139981782,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:23:30.63619766+08:00","start_ts":450139889848025089,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139889848025089,"checkpoint_ts":450139889848025089,"checkpoint_time":"2024-05-31 17:23:26.952"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 48886 6.9 0.3 3166528 1444272 ? Sl 17:23 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 48887 7.1 0.3 3175232 1454328 ? Sl 17:23 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 48888 6.9 0.3 3164476 1444924 ? Sl 17:23 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 48890 6.7 0.3 3168572 1442388 ? Sl 17:23 0:04 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5018 0 --:--:-- --:--:-- --:--:-- 5062 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:24:30.152","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:24:24.903","now_ts":"2024-05-31 17:24:32.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:24:30.152","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:24:24.903","now_ts":"2024-05-31' '17:24:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:24:30.152","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:24:24.903","now_ts":"2024-05-31' '17:24:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 17797 0 --:--:-- --:--:-- --:--:-- 18075 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:24:31.152","puller_resolved_ts":"2024-05-31 17:24:31.152","last_synced_ts":"2024-05-31 17:24:24.903","now_ts":"2024-05-31 17:26:42.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:24:31.152","puller_resolved_ts":"2024-05-31' '17:24:31.152","last_synced_ts":"2024-05-31' '17:24:24.903","now_ts":"2024-05-31' '17:26:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:24:31.152","puller_resolved_ts":"2024-05-31' '17:24:31.152","last_synced_ts":"2024-05-31' '17:24:24.903","now_ts":"2024-05-31' '17:26:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51150, start at 2024-05-31 17:27:03.979881291 +0800 CST m=+4.221927790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51150, start at 2024-05-31 17:27:03.979881291 +0800 CST m=+4.221927790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51053, start at 2024-05-31 17:27:03.945913692 +0800 CST m=+4.240741160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37e587f40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:51150, start at 2024-05-31 17:27:03.979881291 +0800 CST m=+4.221927790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:29:03.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:27:03.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:17:03.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.55165.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450139960453627905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450139960453627905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450139960453627905 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:27:56 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5520655208.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:27:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddd68224-76b7-4eaf-87bb-7025811055cc {"id":"ddd68224-76b7-4eaf-87bb-7025811055cc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfa018a90 ddd68224-76b7-4eaf-87bb-7025811055cc /tidb/cdc/default/default/upstream/7375092854773454107 {"id":7375092854773454107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddd68224-76b7-4eaf-87bb-7025811055cc {"id":"ddd68224-76b7-4eaf-87bb-7025811055cc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfa018a90 ddd68224-76b7-4eaf-87bb-7025811055cc /tidb/cdc/default/default/upstream/7375092854773454107 {"id":7375092854773454107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddd68224-76b7-4eaf-87bb-7025811055cc {"id":"ddd68224-76b7-4eaf-87bb-7025811055cc","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfa018a90 ddd68224-76b7-4eaf-87bb-7025811055cc /tidb/cdc/default/default/upstream/7375092854773454107 {"id":7375092854773454107,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450139960453627905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.55272.out cli changefeed create --start-ts=450139960453627905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375092854773454107,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:27:59.984018342+08:00","start_ts":450139960453627905,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450139960453627905,"checkpoint_ts":450139960453627905,"checkpoint_time":"2024-05-31 17:27:56.291"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 55005 0.8 0.0 2205220 122184 ? Sl 17:27 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717147672369698064.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 55009 0.8 0.0 1870112 124980 ? Sl 17:27 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717147672372985657.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 55096 0.9 0.0 2156844 137604 ? Sl 17:27 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717147672436986642.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep /tmp/tidb_cdc_test/synced_status ++ grep tidb-server ++ xargs kill -9 ++ awk '{print $2}' ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5532 0 --:--:-- --:--:-- --:--:-- 5651 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:28:57.790","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-31 17:28:52.741","now_ts":"2024-05-31 17:28:59.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:28:57.790","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:28:52.741","now_ts":"2024-05-31' '17:28:59.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:28:57.790","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-31' '17:28:52.741","now_ts":"2024-05-31' '17:28:59.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5654 0 --:--:-- --:--:-- --:--:-- 5815 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:31:07.891","puller_resolved_ts":"2024-05-31 17:31:00.941","last_synced_ts":"2024-05-31 17:28:52.741","now_ts":"2024-05-31 17:31:09.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:31:07.891","puller_resolved_ts":"2024-05-31' '17:31:00.941","last_synced_ts":"2024-05-31' '17:28:52.741","now_ts":"2024-05-31' '17:31:09.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:31:07.891","puller_resolved_ts":"2024-05-31' '17:31:00.941","last_synced_ts":"2024-05-31' '17:28:52.741","now_ts":"2024-05-31' '17:31:09.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f6275c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56460, start at 2024-05-31 17:31:36.2964338 +0800 CST m=+4.530436889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f6275c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56460, start at 2024-05-31 17:31:36.2964338 +0800 CST m=+4.530436889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f626a0001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56375, start at 2024-05-31 17:31:36.277858449 +0800 CST m=+4.590578376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37f6275c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:56460, start at 2024-05-31 17:31:36.2964338 +0800 CST m=+4.530436889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:33:36.305 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:31:36.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:21:36.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.60370.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140030425628673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140030425628673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140030425628673 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 17:32:23 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.6041260414.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:32:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62acac84-fc27-4e52-979a-e65b09c51ebb {"id":"62acac84-fc27-4e52-979a-e65b09c51ebb","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfe19ae90 62acac84-fc27-4e52-979a-e65b09c51ebb /tidb/cdc/default/default/upstream/7375094024788419087 {"id":7375094024788419087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62acac84-fc27-4e52-979a-e65b09c51ebb {"id":"62acac84-fc27-4e52-979a-e65b09c51ebb","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfe19ae90 62acac84-fc27-4e52-979a-e65b09c51ebb /tidb/cdc/default/default/upstream/7375094024788419087 {"id":7375094024788419087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62acac84-fc27-4e52-979a-e65b09c51ebb {"id":"62acac84-fc27-4e52-979a-e65b09c51ebb","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdfe19ae90 62acac84-fc27-4e52-979a-e65b09c51ebb /tidb/cdc/default/default/upstream/7375094024788419087 {"id":7375094024788419087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140030425628673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.60476.out cli changefeed create --start-ts=450140030425628673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375094024788419087,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:32:26.891663228+08:00","start_ts":450140030425628673,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450140030425628673,"checkpoint_ts":450140030425628673,"checkpoint_time":"2024-05-31 17:32:23.213"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 13789 0 --:--:-- --:--:-- --:--:-- 13903 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:32:23.213","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 17:32:47.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:32:23.213","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:32:47.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:32:23.213","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:32:47.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri May 31 17:32:58 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Fri May 31 17:32:58 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d000016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61643, start at 2024-05-31 17:33:19.077327983 +0800 CST m=+4.305852645 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d000016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61643, start at 2024-05-31 17:33:19.077327983 +0800 CST m=+4.305852645 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d08001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61545, start at 2024-05-31 17:33:19.082121308 +0800 CST m=+4.363731018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f37fc6d000016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:61643, start at 2024-05-31 17:33:19.077327983 +0800 CST m=+4.305852645 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:35:19.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:33:19.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:23:19.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.65713.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140073072787457 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140073072787457 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:35:06 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.6575365755.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:35:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d095af37-66f2-4b7d-bc3d-a4c1ff62c505 {"id":"d095af37-66f2-4b7d-bc3d-a4c1ff62c505","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdffb0aa0f d095af37-66f2-4b7d-bc3d-a4c1ff62c505 /tidb/cdc/default/default/upstream/7375094461393198943 {"id":7375094461393198943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d095af37-66f2-4b7d-bc3d-a4c1ff62c505 {"id":"d095af37-66f2-4b7d-bc3d-a4c1ff62c505","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdffb0aa0f d095af37-66f2-4b7d-bc3d-a4c1ff62c505 /tidb/cdc/default/default/upstream/7375094461393198943 {"id":7375094461393198943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d095af37-66f2-4b7d-bc3d-a4c1ff62c505 {"id":"d095af37-66f2-4b7d-bc3d-a4c1ff62c505","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fcdffb0aa0f d095af37-66f2-4b7d-bc3d-a4c1ff62c505 /tidb/cdc/default/default/upstream/7375094461393198943 {"id":7375094461393198943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.65824.out cli changefeed create --start-ts=450140073072787457 '--sink-uri=mysql://syncpoint:dGVzdDEyMzQ1Ng==@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: b3f813e3-ddc1-4327-93df-80e59e7556df Info: {"upstream_id":7375094461393198943,"namespace":"default","id":"b3f813e3-ddc1-4327-93df-80e59e7556df","sink_uri":"mysql://syncpoint:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:35:09.625436458+08:00","start_ts":450140073072787457,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450140073072787457,"checkpoint_ts":450140073072787457,"checkpoint_time":"2024-05-31 17:35:05.899"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "mysql.db"="testSync" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.user"="root" "readallfields"="true" "workload"="core" "dotransactions"="false" "scanproportion"="0" "operationcount"="0" "recordcount"="10" ********************************************** Run finished, takes 9.213479ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2047.7, Avg(us): 1766, Min(us): 954, Max(us): 4301, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "mysql.user"="root" "recordcount"="10" "readproportion"="0" "requestdistribution"="uniform" "mysql.db"="testSync" "insertproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "scanproportion"="0" "threadcount"="2" "readallfields"="true" "workload"="core" "dotransactions"="false" "updateproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 6.174728ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2386.3, Avg(us): 1039, Min(us): 510, Max(us): 3486, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "mysql.db"="testSync" "scanproportion"="0" "recordcount"="10" "mysql.user"="root" "readproportion"="0" "mysql.port"="4000" "insertproportion"="0" "workload"="core" "threadcount"="2" "readallfields"="true" "requestdistribution"="uniform" "updateproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.143108ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3777.2, Avg(us): 763, Min(us): 466, Max(us): 1479, 95th(us): 2000, 99th(us): 2000 table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists ____________________________________ *************************** 1. row *************************** primary_ts: 450140073072787457 secondary_ts: 450140075418451981 *************************** 2. row *************************** primary_ts: 450140080937107456 secondary_ts: 450140081041440769 *************************** 3. row *************************** primary_ts: 450140088801427456 secondary_ts: 450140088879284225 *************************** 4. row *************************** primary_ts: 450140096665747456 secondary_ts: 450140096796033025 *************************** 5. row *************************** primary_ts: 450140104530067456 secondary_ts: 450140104660353025 *************************** 6. row *************************** primary_ts: 450140112394387456 secondary_ts: 450140112524673025 *************************** 7. row *************************** primary_ts: 450140120258707456 secondary_ts: 450140120388993025 *************************** 8. row *************************** primary_ts: 450140128123027456 secondary_ts: 450140128279527425 *************************** 9. row *************************** primary_ts: 450140135987347456 secondary_ts: 450140136143847425 skip invalid syncpoint primary_ts: 450140073072787457, first_ddl_ts: 450140075851513883 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:39:11 CST 2024] <<<<<< run test case syncpoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a2f40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:68078, start at 2024-05-31 17:39:39.352384355 +0800 CST m=+4.111666109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a2f40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:68078, start at 2024-05-31 17:39:39.352384355 +0800 CST m=+4.111666109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a1740018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:67983, start at 2024-05-31 17:39:39.262426605 +0800 CST m=+4.100754855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.268 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f3813a2f40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:68078, start at 2024-05-31 17:39:39.352384355 +0800 CST m=+4.111666109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:41:39.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:39:39.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:29:39.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.72134.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140156315566081 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140156315566081 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:40:23 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.7217772179.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:40:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebc234f7-cdc3-4923-8731-0a6286f2885d {"id":"ebc234f7-cdc3-4923-8731-0a6286f2885d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce056e2c8e ebc234f7-cdc3-4923-8731-0a6286f2885d /tidb/cdc/default/default/upstream/7375096101856663701 {"id":7375096101856663701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebc234f7-cdc3-4923-8731-0a6286f2885d {"id":"ebc234f7-cdc3-4923-8731-0a6286f2885d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce056e2c8e ebc234f7-cdc3-4923-8731-0a6286f2885d /tidb/cdc/default/default/upstream/7375096101856663701 {"id":7375096101856663701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebc234f7-cdc3-4923-8731-0a6286f2885d {"id":"ebc234f7-cdc3-4923-8731-0a6286f2885d","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce056e2c8e ebc234f7-cdc3-4923-8731-0a6286f2885d /tidb/cdc/default/default/upstream/7375096101856663701 {"id":7375096101856663701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1627 --:--:-- --:--:-- --:--:-- 1632 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 build mysql_test successfully run all mysql test cases time="2024-05-31T17:40:46+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-31T17:40:46+08:00" level=warning msg="Create new db&{0 0xc0000140a0 0 {0 0} [0xc0002103f0] map[] 0 1 0xc000038240 false map[0xc0002103f0:map[0xc0002103f0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/alter_table.test: ok! 70 test cases passed, take time 20.386969262 s time="2024-05-31T17:41:42+08:00" level=info msg="run test [alter_table] ok" time="2024-05-31T17:41:42+08:00" level=warning msg="Create new db&{0 0xc00031c280 0 {0 0} [0xc00039c000] map[] 0 1 0xc000300a80 false map[0xc00039c000:map[0xc00039c000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/bigint.test: ok! 138 test cases passed, take time 12.847547543 s time="2024-05-31T17:41:55+08:00" level=info msg="run test [bigint] ok" time="2024-05-31T17:41:55+08:00" level=warning msg="Create new db&{0 0xc00031c1c8 0 {0 0} [0xc00016b3b0] map[] 0 1 0xc000300ba0 false map[0xc00016b3b0:map[0xc00016b3b0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/composite_index.test: ok! 12 test cases passed, take time 1.5477149479999999 s time="2024-05-31T17:41:57+08:00" level=info msg="run test [composite_index] ok" time="2024-05-31T17:41:57+08:00" level=warning msg="Create new db&{0 0xc00031c218 0 {0 0} [0xc0004f0090] map[] 0 1 0xc000300e40 false map[0xc0004f0090:map[0xc0004f0090:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.028283609 s time="2024-05-31T17:41:59+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-31T17:41:59+08:00" level=warning msg="Create new db&{0 0xc0003841a0 0 {0 0} [0xc0005be000] map[] 0 1 0xc00038a900 false map[0xc0005be000:map[0xc0005be000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/date_formats.test: ok! 99 test cases passed, take time 5.278123753 s time="2024-05-31T17:42:04+08:00" level=info msg="run test [date_formats] ok" time="2024-05-31T17:42:04+08:00" level=warning msg="Create new db&{0 0xc0004a41c0 0 {0 0} [0xc0001203f0] map[] 0 1 0xc000038ba0 false map[0xc0001203f0:map[0xc0001203f0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/drop.test: ok! 48 test cases passed, take time 8.683631702 s time="2024-05-31T17:42:14+08:00" level=info msg="run test [drop] ok" time="2024-05-31T17:42:14+08:00" level=warning msg="Create new db&{0 0xc000384428 0 {0 0} [0xc0002ba120] map[] 0 1 0xc00038b140 false map[0xc0002ba120:map[0xc0002ba120:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.044471507 s time="2024-05-31T17:42:15+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later table test.finish_mark not exists for 9-th check, retry later table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later table test.finish_mark not exists for 12-th check, retry later table test.finish_mark not exists for 13-th check, retry later table test.finish_mark not exists for 14-th check, retry later table test.finish_mark not exists for 15-th check, retry later table test.finish_mark not exists for 16-th check, retry later table test.finish_mark not exists for 17-th check, retry later table test.finish_mark not exists for 18-th check, retry later table test.finish_mark not exists for 19-th check, retry later table test.finish_mark not exists for 20-th check, retry later table test.finish_mark not exists for 21-th check, retry later table test.finish_mark not exists for 22-th check, retry later table test.finish_mark not exists for 23-th check, retry later table test.finish_mark not exists for 24-th check, retry later table test.finish_mark not exists for 25-th check, retry later table test.finish_mark not exists for 26-th check, retry later table test.finish_mark not exists for 27-th check, retry later table test.finish_mark not exists for 28-th check, retry later table test.finish_mark not exists for 29-th check, retry later table test.finish_mark not exists for 30-th check, retry later table test.finish_mark not exists for 31-th check, retry later table test.finish_mark not exists for 32-th check, retry later table test.finish_mark not exists for 33-th check, retry later table test.finish_mark not exists for 34-th check, retry later table test.finish_mark not exists for 35-th check, retry later table test.finish_mark not exists for 36-th check, retry later table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:43:30 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382322f00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74922, start at 2024-05-31 17:43:53.286766112 +0800 CST m=+4.218241417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.9-3-g61348e865 Edition: Community Git Commit Hash: 61348e865b1b4b46013233aa3e210ea158a364ce Git Branch: HEAD UTC Build Time: 2024-05-29 06:41:10 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-05-29 06:44:05 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382322f00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74922, start at 2024-05-31 17:43:53.286766112 +0800 CST m=+4.218241417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Starting Downstream PD... Release Version: v6.5.9-2-g33d839bdd Edition: Community Git Commit Hash: 33d839bdd7ed98fca9823febbddb43fcdc03a36a Git Branch: release-6.5 UTC Build Time: 2024-05-17 02:30:54 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 627d4a572b4d7d523370e5ed21ee67fee8d798a6 Git Commit Branch: release-6.5 UTC Build Time: 2024-05-10 07:21:43 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-17-g4afee51c07 Edition: Community Git Commit Hash: 4afee51c07a1cf21e054a9d2f7dfd2b4f0c47ee1 Git Branch: release-6.5 UTC Build Time: 2024-05-31 08:15:05 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382321780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74832, start at 2024-05-31 17:43:53.222188844 +0800 CST m=+4.205293350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63f382322f00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-253--ggcnn, pid:74922, start at 2024-05-31 17:43:53.286766112 +0800 CST m=+4.218241417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:45:53.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:43:53.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:33:53.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.79032.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140222814945281 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140222814945281 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:44:37 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.7907279074.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:44:40 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/483a6b17-9162-40e7-93df-eb839840adb7 {"id":"483a6b17-9162-40e7-93df-eb839840adb7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce094f0c8e 483a6b17-9162-40e7-93df-eb839840adb7 /tidb/cdc/default/default/upstream/7375097189162064704 {"id":7375097189162064704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/483a6b17-9162-40e7-93df-eb839840adb7 {"id":"483a6b17-9162-40e7-93df-eb839840adb7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce094f0c8e 483a6b17-9162-40e7-93df-eb839840adb7 /tidb/cdc/default/default/upstream/7375097189162064704 {"id":7375097189162064704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/483a6b17-9162-40e7-93df-eb839840adb7 {"id":"483a6b17-9162-40e7-93df-eb839840adb7","address":"127.0.0.1:8300","version":"v6.5.9-12-g7b0579a42"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce094f0c8e 483a6b17-9162-40e7-93df-eb839840adb7 /tidb/cdc/default/default/upstream/7375097189162064704 {"id":7375097189162064704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.79138.out cli changefeed create --start-ts=450140222814945281 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 9fe60aa9-9fa1-44b8-85c5-6a635f3b3cfd Info: {"upstream_id":7375097189162064704,"namespace":"default","id":"9fe60aa9-9fa1-44b8-85c5-6a635f3b3cfd","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:44:40.843010485+08:00","start_ts":450140222814945281,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-12-g7b0579a42","resolved_ts":450140222814945281,"checkpoint_ts":450140222814945281,"checkpoint_time":"2024-05-31 17:44:37.120"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:45:39 CST 2024] <<<<<< run test case tiflash success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-253/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE